diff --git a/llm-server/llm/ansible/playbooks/switch-environment.yml b/llm-server/llm/ansible/playbooks/switch-environment.yml new file mode 100644 index 0000000..8f50766 --- /dev/null +++ b/llm-server/llm/ansible/playbooks/switch-environment.yml @@ -0,0 +1,63 @@ +--- +# Performs the exclusive AI environment switch (GNAI-008 section 7.3). +# +# Usage: +# ansible-playbook -i inventories/ playbooks/switch-environment.yml \ +# -e ai_environment_target=production \ +# --private-key ~/.ssh/id_ed25519_ghostnet_ai_ansible_deploy +# +# Non-fallback targets require a maintenance window in minutes: +# -e ai_environment_target=testing -e ai_environment_window_minutes=180 +# +# The inventory only provides the connection; the environment mapping +# lives in the ai_environment_switch role defaults. The recovery service +# and timer return to the fallback automatically once a window expired, +# independently of this playbook or any CI runner. +- name: Perform the exclusive AI environment switch + hosts: llm_servers + become: true + gather_facts: false + + roles: + - role: ai_environment_switch + + tasks: + - name: Require a switch target + ansible.builtin.assert: + that: + - ai_environment_target is defined + - ai_environment_target in ai_environment_definitions + fail_msg: >- + Set -e ai_environment_target=production|testing (must be a known + environment). + + - name: Require a maintenance window for non-fallback targets + ansible.builtin.assert: + that: + - ai_environment_target == ai_environment_fallback + or ai_environment_window_minutes is defined + fail_msg: >- + Switching to {{ ai_environment_target | default('') }} + requires -e ai_environment_window_minutes=. + + - name: Flush role handlers before switching + ansible.builtin.meta: flush_handlers + + - name: Execute the environment switch + ansible.builtin.command: + argv: >- + {{ [ai_environment_switch_script, 'switch-to', ai_environment_target] + + ([ai_environment_window_minutes | string] + if (ai_environment_window_minutes is defined + and ai_environment_window_minutes | string | length > 0) + else []) }} + register: ai_environment_switch_result + changed_when: >- + 'switched to' in ai_environment_switch_result.stdout + or 'already active and running' in ai_environment_switch_result.stdout + failed_when: ai_environment_switch_result.rc != 0 + + - name: Report the switch result + ansible.builtin.debug: + msg: "{{ ai_environment_switch_result.stdout_lines + + ai_environment_switch_result.stderr_lines }}" diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/README.md b/llm-server/llm/ansible/roles/ai_environment_switch/README.md new file mode 100644 index 0000000..b96c7bc --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/README.md @@ -0,0 +1,25 @@ +# ai_environment_switch + +Exclusive switch between the AI environments (GNAI-008 section 7.3, +GNAI-001 milestone 4). Deploys: + +- `gnai-environment-switch` (privileged host script): flock-locked + transitions with stop-before-start, GPU release proof (`/dev/kfd`, + render nodes per environment UID), health wait and atomic state file + (`/var/lib/ghostnet-ai/environment-state`). +- `/etc/ghostnet-ai/environment-switch.conf`: the environment mapping + (accounts, UIDs, unit prefixes, active units) from `defaults/main.yml`. +- `gnai-environment-recovery.service` + `.timer`: idempotent + reconciliation — after host boots it brings the state-active + environment back up; every minute it returns to the fallback + (`production`) once a maintenance window has expired. This covers + runner aborts and lost CI without production secrets in any CI path. + +Subcommands of the script: `status`, `switch-to [window-minutes]` +and `recover`. Non-fallback switches require a maintenance window and +are validated; the window end is recorded in the state file. + +The switch itself is executed via `playbooks/switch-environment.yml` +(`-e ai_environment_target=… [-e ai_environment_window_minutes=…]`). +Unit names and accounts must stay in sync with the per-environment +`inference_engine` group_vars. diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/defaults/main.yml b/llm-server/llm/ansible/roles/ai_environment_switch/defaults/main.yml new file mode 100644 index 0000000..4db77b1 --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/defaults/main.yml @@ -0,0 +1,51 @@ +--- +# Exclusive AI environment switch (GNAI-008 section 7.3). +# +# A privileged host script performs the switch (lock, stop-before-start, +# GPU release proof, timeouts, recovery); this role deploys the script, +# its environment mapping, the recovery service and the recovery timer. +# The state file is the queryable status source for external callers +# (fleet tooling, future CI). + +ai_environment_switch_script: /usr/local/sbin/gnai-environment-switch +ai_environment_switch_config: /etc/ghostnet-ai/environment-switch.conf +ai_environment_state_file: /var/lib/ghostnet-ai/environment-state +ai_environment_state_dir: /var/lib/ghostnet-ai + +# Environment mapping (accounts, unit prefixes and the desired active unit +# per environment as of the latest deployment). Unit prefixes follow the +# inference_engine group_vars of each environment. +ai_environment_definitions: + production: + account: gnai-production + uid: 980 + unit_prefix: gnai-production-llama + active_unit: gnai-production-llama-swift15 + testing: + account: gnai-testing + uid: 981 + unit_prefix: gnai-testing-llama + active_unit: gnai-testing-llama-swift15 + +# Which environment is the stable fallback (GNAI-008 section 7.3): the +# recovery service starts it after host boots and the recovery timer +# returns to it once a maintenance window has expired. +ai_environment_fallback: production + +# Timeouts for one switch operation, in seconds. +ai_environment_stop_timeout: 180 +ai_environment_gpu_release_timeout: 60 +ai_environment_start_timeout: 240 +ai_environment_health_timeout: 300 +ai_environment_health_interval: 5 + +# Maintenance windows: a switch to a non-fallback environment records a +# window end timestamp in the state file; the recovery timer returns to +# the fallback automatically once the window has expired. The window +# length is provided per invocation by the caller (playbook/CI), not +# defaulted here. + +# Recovery service/timer (M4 recovery: host boot and aborted runs). +ai_environment_recovery_service: gnai-environment-recovery +ai_environment_recovery_timer: gnai-environment-recovery +ai_environment_recovery_timer_interval: "1 min" diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/handlers/main.yml b/llm-server/llm/ansible/roles/ai_environment_switch/handlers/main.yml new file mode 100644 index 0000000..c6576bd --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/handlers/main.yml @@ -0,0 +1,35 @@ +--- +- name: Verify the environment switch configuration + ansible.builtin.command: + argv: + - /bin/bash + - -n + - "{{ ai_environment_switch_config }}" + changed_when: false + failed_when: false + when: not ansible_check_mode + +- name: Verify the environment script syntax + ansible.builtin.command: + argv: + - /bin/bash + - -n + - "{{ ai_environment_switch_script }}" + register: ai_environment_switch_syntax + changed_when: false + failed_when: ai_environment_switch_syntax.rc != 0 + when: not ansible_check_mode + +- name: Reload the systemd daemon + ansible.builtin.systemd_service: + daemon_reload: true + when: not ansible_check_mode + +- name: Restart the environment recovery units + ansible.builtin.systemd_service: + name: "{{ item }}" + state: restarted + loop: + - "{{ ai_environment_recovery_service }}.service" + - "{{ ai_environment_recovery_timer }}.timer" + when: not ansible_check_mode diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/tasks/main.yml b/llm-server/llm/ansible/roles/ai_environment_switch/tasks/main.yml new file mode 100644 index 0000000..d57c5c4 --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/tasks/main.yml @@ -0,0 +1,101 @@ +--- +# Deploys the exclusive AI environment switch (GNAI-008 section 7.3): +# the privileged host script, its environment mapping, the recovery +# service and the recovery timer. The switch itself is executed through +# playbooks/switch-environment.yml. + +- name: Verify the environment switch declarations + ansible.builtin.assert: + that: + - ai_environment_fallback in ai_environment_definitions + - ai_environment_definitions | length >= 2 + - >- + ai_environment_definitions.values() | map(attribute='account') | unique + | list | length == ai_environment_definitions | length + - >- + ai_environment_definitions.values() | map(attribute='uid') | unique + | list | length == ai_environment_definitions | length + - >- + ai_environment_definitions.values() | map(attribute='unit_prefix') + | unique | list | length == ai_environment_definitions | length + - ai_environment_gpu_release_timeout | int > 0 + - ai_environment_start_timeout | int > 0 + - ai_environment_health_timeout | int > 0 + - ai_environment_health_interval | int > 0 + fail_msg: AI environment switch declarations are incomplete or ambiguous. + +- name: Verify every environment account exists on the host + ansible.builtin.getent: + database: passwd + key: "{{ item.value.account }}" + loop: "{{ ai_environment_definitions | dict2items }}" + loop_control: + label: "{{ item.value.account }}" + +- name: Verify the account UID matches the declaration + ansible.builtin.assert: + that: + - getent_passwd[item.value.account][1] | int == item.value.uid | int + fail_msg: >- + UID mismatch for {{ item.value.account }}: declared {{ item.value.uid }}, + host reports {{ getent_passwd[item.value.account][1] }}. + loop: "{{ ai_environment_definitions | dict2items }}" + loop_control: + label: "{{ item.value.account }}" + +- name: Ensure the host configuration directory exists + ansible.builtin.file: + path: "{{ ai_environment_switch_config | dirname }}" + state: directory + mode: "0755" + +- name: Install the environment switch configuration + ansible.builtin.template: + src: environment-switch.conf.j2 + dest: "{{ ai_environment_switch_config }}" + mode: "0644" + notify: + - Verify the environment switch configuration + +- name: Install the environment switch script + ansible.builtin.template: + src: gnai-environment-switch.sh.j2 + dest: "{{ ai_environment_switch_script }}" + mode: "0750" + notify: + - Verify the environment script syntax + +- name: Install the environment recovery service + ansible.builtin.template: + src: gnai-environment-recovery.service.j2 + dest: "/etc/systemd/system/{{ ai_environment_recovery_service }}.service" + mode: "0644" + notify: + - Reload the systemd daemon + - Restart the environment recovery units + +- name: Install the environment recovery timer + ansible.builtin.template: + src: gnai-environment-recovery.timer.j2 + dest: "/etc/systemd/system/{{ ai_environment_recovery_timer }}.timer" + mode: "0644" + notify: + - Reload the systemd daemon + - Restart the environment recovery units + +- name: Enable and start the environment recovery timer + ansible.builtin.systemd_service: + name: "{{ ai_environment_recovery_timer }}.timer" + enabled: true + state: started + when: not ansible_check_mode + +- name: Verify the environment switch script answers a status query + ansible.builtin.command: + argv: + - "{{ ai_environment_switch_script }}" + - status + register: ai_environment_switch_status + changed_when: false + failed_when: ai_environment_switch_status.rc != 0 + when: not ansible_check_mode diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/templates/environment-switch.conf.j2 b/llm-server/llm/ansible/roles/ai_environment_switch/templates/environment-switch.conf.j2 new file mode 100644 index 0000000..c2f32e1 --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/templates/environment-switch.conf.j2 @@ -0,0 +1,16 @@ +# {{ ansible_managed }} +# Environment mapping for gnai-environment-switch (GNAI-008 7.3). +# Deployed by the ai_environment_switch role; do not edit on the host. + +ENVIRONMENTS="{% for env in ai_environment_definitions.keys() | sort %}{{ env }}{% if not loop.last %} {% endif %}{% endfor %}" +FALLBACK="{{ ai_environment_fallback | default('production') }}" +TIMEOUT_GPU_RELEASE={{ ai_environment_gpu_release_timeout }} +TIMEOUT_START_AND_HEALTH={{ [ai_environment_start_timeout, ai_environment_health_timeout] | max }} +HEALTH_INTERVAL={{ ai_environment_health_interval }} + +{% for env, definition in ai_environment_definitions.items() %} +{{ env }}_account={{ definition.account }} +{{ env }}_uid={{ definition.uid }} +{{ env }}_unit_prefix={{ definition.unit_prefix }} +{{ env }}_active_unit={{ definition.active_unit }} +{% endfor %} diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.service.j2 b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.service.j2 new file mode 100644 index 0000000..eeefeb6 --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.service.j2 @@ -0,0 +1,12 @@ +# {{ ansible_managed }} +[Unit] +# M4 recovery path (GNAI-008 7.3): reconciles the AI environment state +# after host boots and aborted runs without any CI involvement. The +# timer {{ ai_environment_recovery_timer }}.timer calls the same +# operation every minute (expired windows return to the fallback). +Description=Ghostnet AI environment recovery +After=multi-user.target + +[Service] +Type=oneshot +ExecStart={{ ai_environment_switch_script }} recover diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.timer.j2 b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.timer.j2 new file mode 100644 index 0000000..8ead27c --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-recovery.timer.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} +[Unit] +# Periodic reconciliation of the AI environment state; see the +# gnai-environment-recovery service unit for the operation. +Description=Ghostnet AI environment recovery timer + +[Timer] +OnBootSec=2 min +OnUnitActiveSec={{ ai_environment_recovery_timer_interval }} +AccuracySec=30s + +[Install] +WantedBy=timers.target diff --git a/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-switch.sh.j2 b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-switch.sh.j2 new file mode 100644 index 0000000..4c39910 --- /dev/null +++ b/llm-server/llm/ansible/roles/ai_environment_switch/templates/gnai-environment-switch.sh.j2 @@ -0,0 +1,291 @@ +#!/bin/bash +# gnai-environment-switch — exclusive AI environment switch +# (GNAI-008 section 7.3). +# +# Performs the ordered environment transition: takes an exclusive lock, +# stops the running environment (stop-before-start), proves the GPU +# release, starts the target environment's active unit and waits for its +# health endpoint. Every non-fallback target records a maintenance window +# end in the state file; the recovery service and timer return to the +# fallback once that window has expired. +# +# Subcommands: +# status print the state file and the actually +# running units per environment +# switch-to [minutes] switch to ; [minutes] records a +# maintenance window (required and validated +# for non-fallback targets) +# recover idempotent reconciliation: start the state +# active environment when it is down; return +# to the fallback once its window expired +# +# Exit codes: 0 ok; 1 usage error; 2 unknown environment or config; +# 10 switch already in progress (locked); 11 stop or GPU release timeout; +# 12 start or health timeout. + +set -u +cd / + +CONFIG="{{ ai_environment_switch_config }}" +STATE="{{ ai_environment_state_file }}" +LOCK_DIR=/run/ghostnet-ai +LOCK="$LOCK_DIR/environment-switch.lock" + +# shellcheck source=/dev/null +[ -r "$CONFIG" ] || { echo "config missing: $CONFIG" >&2; exit 2; } +# shellcheck disable=SC1090 +. "$CONFIG" + +: "${ENVIRONMENTS:?}" "${FALLBACK:?}" "${TIMEOUT_GPU_RELEASE:?}" \ + "${TIMEOUT_START_AND_HEALTH:?}" "${HEALTH_INTERVAL:?}" + +env_value() { + # $1 = environment, $2 = key; prints _ from the config + local name="${1}_${2}" + echo "${!name-}" +} + +epoch() { date +%s; } + +write_state() { + # $1.. = key=value pairs (atomic write) + local tmp="$STATE.tmp" + : > "$tmp" + printf '# managed by gnai-environment-switch\n' >> "$tmp" + for kv in "$@"; do printf '%s\n' "$kv" >> "$tmp"; done + mv "$tmp" "$STATE" +} + +state_get() { + # $1 = key; prints its value or empty + sed -n "s/^$1=//p" "$STATE" 2>/dev/null | head -n1 +} + +as_env() { + # $1 = environment, rest = command; runs as the environment account + local account uid home + account=$(env_value "$1" account) + uid=$(env_value "$1" uid) + home=$(getent passwd "$account" | cut -d: -f6) + runuser -u "$account" -- \ + env HOME="$home" XDG_RUNTIME_DIR="/run/user/$uid" "${@:2}" +} + +running_units() { + # $1 = environment; prints the running engine unit names + local status + status=$(as_env "$1" systemctl --user is-active \ + "$(env_value "$1" active_unit).service" 2>/dev/null) || status=inactive + if [ "$status" = "active" ]; then + echo "$(env_value "$1" active_unit).service" + fi +} + +gpu_holders() { + # $1 = uid; prints PIDs of that uid holding /dev/kfd or a render node + local pid fd target + for pid in /proc/[0-9]*; do + [ "$(stat -Lc %u "$pid" 2>/dev/null)" = "$1" ] || continue + for fd in "$pid"/fd/*; do + target=$(readlink "$fd" 2>/dev/null) || continue + case "$target" in + /dev/kfd|/dev/dri/*) echo "${pid##*/}"; break ;; + esac + done + done | sort -u +} + +wait_gpu_release() { + # $1 = uid, $2 = timeout seconds + local waited=0 + while [ "$waited" -lt "$2" ]; do + [ -z "$(gpu_holders "$1")" ] && return 0 + sleep 5 + waited=$((waited + 5)) + done + echo "GPU release timeout for uid $1 (holders: $(gpu_holders "$1" | tr '\n' ' '))" >&2 + return 11 +} + +wait_health() { + # $1 = environment, $2 = timeout seconds, $3 = interval seconds + local waited=0 + while [ "$waited" -lt "$2" ]; do + if as_env "$1" podman exec "$(env_value "$1" active_unit)" \ + sh -c "curl -fsS -o /dev/null http://127.0.0.1:8080/health" \ + >/dev/null 2>&1; then + return 0 + fi + sleep "$3" + waited=$((waited + $3)) + done + return 12 +} + +lock_or_exit() { + mkdir -p "$LOCK_DIR" + exec 9>"$LOCK" + flock -n 9 || { echo "environment switch already in progress" >&2; exit 10; } +} + +stop_environment() { + # $1 = environment; ordered stop of every running engine unit + local unit + for unit in $(as_env "$1" systemctl --user list-units \ + "$(env_value "$1" unit_prefix)-*.service" --no-legend --plain \ + 2>/dev/null | awk '{print $1}'); do + as_env "$1" systemctl --user stop "$unit" 2>/dev/null + done + sleep 2 +} + +start_active_unit() { + # $1 = environment + as_env "$1" systemctl --user start "$(env_value "$1" active_unit).service" +} + +switch_to() { + # $1 = target environment, $2 = window minutes (optional) + local target="$1" window_minutes="$2" other uid + local active unit refresh_window="" + + case " $ENVIRONMENTS " in *" $target "*) ;; *) + echo "unknown environment: $target" >&2; exit 2 ;; + esac + + lock_or_exit + + if [ "$target" != "$FALLBACK" ]; then + case "$window_minutes" in + ''|*[!0-9]*) + echo "refusing non-fallback switch without a maintenance window" >&2 + exit 1 ;; + esac + refresh_window=yes + fi + + unit=$(env_value "$target" active_unit) + [ -n "$unit" ] || { echo "no active unit configured for $target" >&2; exit 2; } + + active=$(state_get active) + if [ "$active" = "$target" ] && [ -n "$(running_units "$target")" ]; then + # Refresh the maintenance window when the target already runs. + if [ -n "$refresh_window" ]; then + write_state "active=$target" "active_unit=$unit" \ + "active_since=$(epoch)" \ + "window_until=$(( $(epoch) + window_minutes * 60 ))" + fi + echo "environment $target already active and running (idempotent)" + return 0 + fi + + # Stop every OTHER environment (stop-before-start), prove GPU release. + for other in $ENVIRONMENTS; do + [ "$other" = "$target" ] && continue + if [ -n "$(running_units "$other")" ]; then + uid=$(env_value "$other" uid) + stop_environment "$other" + wait_gpu_release "$uid" "$TIMEOUT_GPU_RELEASE" || exit 11 + fi + done + + start_active_unit "$target" + + if ! wait_health "$target" "$TIMEOUT_START_AND_HEALTH" "$HEALTH_INTERVAL"; then + echo "target $target failed to become healthy" >&2 + if [ "$target" != "$FALLBACK" ]; then + echo "rolling back to $FALLBACK" >&2 + stop_environment "$target" 2>/dev/null || true + wait_gpu_release "$(env_value "$target" uid)" \ + "$TIMEOUT_GPU_RELEASE" 2>/dev/null || true + start_active_unit "$FALLBACK" 2>/dev/null || true + wait_health "$FALLBACK" "$TIMEOUT_START_AND_HEALTH" \ + "$HEALTH_INTERVAL" 2>/dev/null || true + fi + exit 12 + fi + + if [ -n "$refresh_window" ]; then + write_state "active=$target" "active_unit=$unit" \ + "active_since=$(epoch)" \ + "window_until=$(( $(epoch) + window_minutes * 60 ))" + else + write_state "active=$target" "active_unit=$unit" \ + "active_since=$(epoch)" "window_until=" + fi + echo "switched to $target (unit $unit)" + return 0 +} + +cmd_status() { + local env uid units + echo "state file: $STATE" + if [ -r "$STATE" ]; then sed 's/^/ /' "$STATE"; else echo " (unset)"; fi + for env in $ENVIRONMENTS; do + units=$(running_units "$env") + uid=$(env_value "$env" uid) + echo "runtime: $env uid=$uid units=${units:-none} gpu=$(gpu_holders "$uid" | tr '\n' ' ')" + done + return 0 +} + +cmd_recover() { + local active window_until now other + lock_or_exit + + if [ ! -r "$STATE" ]; then + # No state yet: the fallback is the defined stable state. + if [ -z "$(running_units "$FALLBACK")" ]; then + start_active_unit "$FALLBACK" || return 0 + fi + write_state "active=$FALLBACK" \ + "active_unit=$(env_value "$FALLBACK" active_unit)" \ + "active_since=$(epoch)" "window_until=" + return 0 + fi + + active=$(state_get active) + window_until=$(state_get window_until) + now=$(epoch) + + if [ "$active" != "$FALLBACK" ]; then + if [ -z "$window_until" ] || [ "$now" -ge "$window_until" ]; then + echo "maintenance window over, returning to $FALLBACK" + switch_to "$FALLBACK" "" &2 + stop_environment "$other" + wait_gpu_release "$(env_value "$other" uid)" \ + "$TIMEOUT_GPU_RELEASE" || true + fi + done + start_active_unit "$FALLBACK" || true + fi + return 0 +} + +main() { + local cmd="${1:-}"; shift || true + case "$cmd" in + status) cmd_status "$@" ;; + switch-to) switch_to "$@" ;; + recover) cmd_recover "$@" ;; + *) echo "usage: $0 {status|switch-to [window-minutes]|recover}" >&2 + exit 1 ;; + esac +} + +main "$@"