diff --git a/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md b/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md index 717d07c..34691d5 100644 --- a/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md +++ b/docs/platform/llm-server/ai/GNAI-010-TECH_Llama-Serving.md @@ -35,6 +35,7 @@ | 2026-11-04 | 0.16 | spektr / Mistral Vibe | Slot-Tauschprobe korrigiert die Fehleranalyse (Abschnitt 10.10): Der thermische Defekt folgt dem Slot 03:00.0, nicht der Karte (verdächtige Karte im Slot 07:00.0 stabil bei ~60 °C; unauffällige Karte im Slot 03:00.0 bei 86–88 °C nach 5,5 Min bei gleicher Leistung). Beide Slots liegen über dem 01.11.-Referenzniveau – Verdacht auf Luftführungs-Regression am Gehäuse (Filter/Gehäuselüfter/Kabel). Karten-RMA zurückgestellt; Gehäuse-Prüfung und GNMON-001-Anschluss des KI-Servers als Folgeaktion festgelegt. | | 2026-11-04 | 0.17 | spektr / Mistral Vibe | Deckel-Einflussmessung ergänzt (Abschnitt 10.10): Bei offenem Deckel 89 °C Junction im Slot 03:00.0 nach 6,75 Min statt 86–88 °C nach 5,5 Min geschlossen – Deckel liefert nur ~1–1,5 Min Verzögerung. Hypothese gehäuseweite Luftführungs-Revision verworfen; Engpass ist das Karten-Sandwich lokal. Konfigurative Hebel (PWM-Anhebung, Tensor-Split, Power Cap) als nächster Testschritt nach GNMON-001-Anschluss festgehalten. | | 2026-11-05 | 0.18 | spektr / Mistral Vibe | PWM-Durchgriffstest ergänzt (Abschnitt 10.10): manueller Lüfter-PWM 255 auf der Karte im Slot 03:00.0 ergibt 3.149 RPM und ein stabiles Junction-Plateau von 76 °C (Peaks 80 °C) bei unverändertem Durchsatz (18,04 t/s) – Vorläufe brachen bei 86–93 °C ab. Werkkurve als Engpass identifiziert; Benchmarks unter Override thermisch wieder zulässig, eigenes Lüfterkurven-Service als Dauerlösung festgelegt, M5-Monitoring als live verifiziert dokumentiert. Override nach Test auf Auto zurückgesetzt. | +| 2026-11-05 | 0.19 | spektr / Mistral Vibe | Dauerlösung deployt (Abschnitt 10.10): Rolle `gpu_fan_control` im Host-Projekt installiert den Systemdienst `gnai-gpu-fan-control` (PCI-stabile Adressierung beider Karten, Junction-Kurve 50/58/66/72/76 °C → 0/80/140/200/255 PWM, 5s-Zyklus, fallende Hysterese, EXIT-Failsafe auf Werksteuerung, Re-Assert des manuellen Modus je Zyklus). Preview/Deploy/Idempotenz bestanden, Kurzlasttest regelt 67–72 °C; BFCL-Restserie läuft wieder unter Lüfterschutz. | --- @@ -911,16 +912,41 @@ lösung (eigene Lüfterkurve mit Hysterese als kleines Service, da die Werkurve träge resp. defekt erscheint) ist Folgeaufgabe. Der PWM-Override wurde nach dem Test auf `pwm1_enable=2` (Auto) zurückgesetzt. +#### Dauerlösung: Lüfterkurven-Service (05.11.2026) + +Der manuelle Override ist durch die Rolle `gpu_fan_control` im Host-Projekt +(`llm-server/host/ansible/roles/gpu_fan_control/`, verdrahtet in +`playbooks/host.yml`) ersetzt: der Systemdienst `gnai-gpu-fan-control` +steuert den Lüfter-PWM beider Karten (PCI-stabil adressiert, keine +DRM-Indizierung) aus der Junction-Temperatur über eine lineare Kurve +(Anker 50 °C→0, 58→80, 66→140, 72→200, 76→255 PWM; 5-Sekunden-Zyklus, +fallende Hysterese von zwei Quantisierungsschritten à 16). Designmerkmale: + +- a) Der EXIT-Trap stellt `pwm1_enable=2` (Werksteuerung) auf allen Karten + zurück; ein gestoppter Dienst friert kein PWM ein. Der systemd-Stop + nutzt das `sleep & wait`-Muster, damit der Trap auf SIGTERM promptly + läuft. +- b) Der Dienst re-assertiert `pwm1_enable=1` jeden Zyklus; ein + amdgpu-Reload kann die Werkkurve nicht unbemerkt zurückholen. +- c) Kurvenanker sind Rollen-Defaults, anpassbar ohne Skriptänderung. + +Abnahme: Preview `--check --diff` sauber, produktiver Deploy, Idempotenz +bestätigt (`changed=0`), Dienst aktiv auf beiden Karten +(`pwm1_enable=1`, Idle PWM 0), Kurzlasttest zeigt Regelung 67–72 °C bei +PWM 144–192 (2.757 RPM) auf der Karte im Slot 03:00.0; die Karte 07:00.0 +moduliert um ihre Kurvenanker — kein kritischer Zustand. Damit laufen alle +weiteren Benchmark-Serien standardmäßig unter dem Lüfterschutz; der +PWM-Durchgriffstest von oben ist obsolet. + Konsequenzen und Folgeaktionen: -- a) Die Benchmark-Serien (BFCL-Restserie Swift 1.0 und Unsloth) sind unter - vollem Lüfterdurchgriff (pwm 255) thermisch wieder zulässig; der - Override ist pro Lauf manuell zu setzen und danach auf Auto - zurückzusetzen, bis eine Dauerlösung besteht. -- b) Für den Dauerbetrieb wird ein eigenes Lüfterkurven-Service - (Junction→PWM mit Hysterese) angestrebt, da die Werkkurve träge bzw. - defekt erscheint (Stand 05.11.2026); bis dahin läuft produktive Last - nur mit Override. +- a) Die Benchmark-Serien (BFCL-Restserie Swift 1.0 und Unsloth) sind + wieder aufgenommen und laufen ab 05.11.2026 standardmäßig unter dem + Lüfterschutz des Dienstes (kein manueller Override mehr nötig). +- b) Dauerlösung umgesetzt: Rolle `gpu_fan_control`/Dienst + `gnai-gpu-fan-control` (siehe Abschnitt Dauerlösung oben); Kurven- + Anker sind Rollen-Defaults und bei Bedarf ohne Skriptänderung + nachzusteuern. - c) Die dauerhafte Sensorüberwachung mit Schwellwert-Alarm wurde mit GNMON-001 M5 umgesetzt (K1-Collector + Server-VictoriaMetrics, verifiziert 05.11.2026 über SSH-Tunnel-Abfrage durch den Agenten; diff --git a/llm-server/host/ansible/playbooks/host.yml b/llm-server/host/ansible/playbooks/host.yml index e1cd9c7..9f0cf5d 100644 --- a/llm-server/host/ansible/playbooks/host.yml +++ b/llm-server/host/ansible/playbooks/host.yml @@ -13,3 +13,4 @@ - role: podman_runtime - role: workload_accounts - role: amd_gpu_access + - role: gpu_fan_control diff --git a/llm-server/host/ansible/roles/gpu_fan_control/README.md b/llm-server/host/ansible/roles/gpu_fan_control/README.md new file mode 100644 index 0000000..b1000b6 --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/README.md @@ -0,0 +1,22 @@ +# gpu_fan_control + +Installs `gnai-gpu-fan-control`, a root system service that drives the fan +PWM of the configured amdgpu cards (PCI addresses, stable across DRM +renumbering) from their junction temperature. + +Rationale and measurements: GNAI-010 section 10.10 — the factory fan +curves ramp far too slowly for the cramped dual-GPU layout (SW CTF +incident 2026-11-03); with a junction based curve the hot slot holds a +verified plateau of 76 celsius under sustained inference load. + +Behaviour: + +- Linear interpolation between the curve anchors from `defaults/main.yml` + (idle below 50 celsius rests at PWM 0, full duty is reached by 76 + celsius). +- Targets are raised immediately, lowered only after falling at least + two PWM steps below the current value (falling hysteresis). +- The service re-asserts `pwm1_enable=1` every cycle so an amdgpu reload + cannot silently restore factory control mid-load. +- The EXIT trap restores factory control (`pwm1_enable=2`) on every + configured card, so a stopped service never leaves a frozen PWM. diff --git a/llm-server/host/ansible/roles/gpu_fan_control/defaults/main.yml b/llm-server/host/ansible/roles/gpu_fan_control/defaults/main.yml new file mode 100644 index 0000000..239e557 --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/defaults/main.yml @@ -0,0 +1,34 @@ +# GPU fan curve control (GNAI-010 section 10.10, decision of 2026-11-05). +# The factory fan curves of the amdgpu cards ramp far too slowly for the +# cramped dual-GPU layout (SW CTF incident 2026-11-03). This role installs a +# small system service that drives the fan PWM of the configured cards from +# their junction temperature and restores factory control on exit. + +gpu_fan_control_install_dir: /usr/local/lib/ghostnet-ai +gpu_fan_control_script: gnai-gpu-fan-control.sh +gpu_fan_control_service: gnai-gpu-fan-control + +# PCI addresses of the amdgpu cards to control. PCI addressing is stable +# across DRM renumbering; the slot layout (card sandwich) is the thermal +# constraint, not the specific card. +gpu_fan_control_pci_addresses: + - "0000:03:00.0" + - "0000:07:00.0" + +# Junction temperature (celsius) to PWM (0..255) curve anchors. The +# controller interpolates linearly between anchors. Below the first anchor +# the fans rest at PWM 0 (idle zero-RPM), by 76 celsius they reach full +# duty, matching the verified PWM override plateau (GNAI-010 10.10). +gpu_fan_control_curve: + - { temp_c: 50, pwm: 0 } + - { temp_c: 58, pwm: 80 } + - { temp_c: 66, pwm: 140 } + - { temp_c: 72, pwm: 200 } + - { temp_c: 76, pwm: 255 } + +# Control behaviour: the PWM target is raised immediately but lowered only +# after the target falls at least pwm_step * 2 below the current value +# (falling hysteresis against oscillation). Targets are quantized to +# pwm_step to avoid constant micro changes. +gpu_fan_control_pwm_step: 16 +gpu_fan_control_interval_seconds: 5 diff --git a/llm-server/host/ansible/roles/gpu_fan_control/handlers/main.yml b/llm-server/host/ansible/roles/gpu_fan_control/handlers/main.yml new file mode 100644 index 0000000..3902fc2 --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Reload the systemd daemon + ansible.builtin.systemd_service: + daemon_reload: true + when: not ansible_check_mode + +- name: Restart the GPU fan curve control service + ansible.builtin.systemd_service: + name: "{{ gpu_fan_control_service }}" + state: restarted + when: not ansible_check_mode diff --git a/llm-server/host/ansible/roles/gpu_fan_control/tasks/main.yml b/llm-server/host/ansible/roles/gpu_fan_control/tasks/main.yml new file mode 100644 index 0000000..b0e6278 --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/tasks/main.yml @@ -0,0 +1,117 @@ +--- +# GPU fan curve control (GNAI-010 section 10.10). Installs a system +# service that replaces the too-slow factory fan curves of the amdgpu +# cards with a junction-temperature based curve. Host scope: this role +# is applied only to the dedicated AI server and requires the amdgpu +# driver for every configured card. + +- name: Verify GPU fan control declarations + ansible.builtin.assert: + that: + - gpu_fan_control_pci_addresses | length > 0 + - gpu_fan_control_pci_addresses | unique | length == gpu_fan_control_pci_addresses | length + - gpu_fan_control_pci_addresses | select('match', '^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7a-f]$') | list | length == gpu_fan_control_pci_addresses | length + - gpu_fan_control_curve | length >= 2 + - gpu_fan_control_pwm_step | int > 0 + - gpu_fan_control_pwm_step | int <= 128 + - gpu_fan_control_interval_seconds | int >= 2 + fail_msg: GPU fan control declarations are invalid. + +- name: Verify fan curve anchors are ascending and in PWM range + ansible.builtin.assert: + that: + - gpu_fan_control_curve[0].temp_c | int < gpu_fan_control_curve[-1].temp_c | int + - >- + gpu_fan_control_curve | map(attribute='temp_c') | list + | sort | list == gpu_fan_control_curve | map(attribute='temp_c') | list + - >- + gpu_fan_control_curve | map(attribute='pwm') | map('int') + | select('>', 255) | list | length == 0 + - >- + gpu_fan_control_curve | map(attribute='pwm') | map('int') + | select('<', 0) | list | length == 0 + fail_msg: Fan curve anchors must be unique, ascending in temperature and within 0..255 PWM. + +- name: Verify every configured card is bound to the amdgpu driver + ansible.builtin.stat: + path: "/sys/bus/pci/devices/{{ item }}/driver" + get_checksum: false + get_mime: false + loop: "{{ gpu_fan_control_pci_addresses }}" + loop_control: + label: "{{ item }}" + register: gpu_fan_control_driver_links + +- name: Verify the amdgpu driver symlink resolves to the amdgpu module + ansible.builtin.assert: + that: + - gpu_fan_control_driver_links.results | map(attribute='stat') | map(attribute='exists') | select | list | length == gpu_fan_control_pci_addresses | length + - gpu_fan_control_driver_links.results | map(attribute='stat.lnk_source') | select('search', '/amdgpu$') | list | length == gpu_fan_control_pci_addresses | length + fail_msg: >- + Every configured card must expose an amdgpu driver binding in sysfs. + +- name: Ensure the install directory exists + ansible.builtin.file: + path: "{{ gpu_fan_control_install_dir }}" + state: directory + mode: "0755" + +- name: Install the GPU fan curve control script + ansible.builtin.template: + src: gnai-gpu-fan-control.sh.j2 + dest: "{{ gpu_fan_control_install_dir }}/{{ gpu_fan_control_script }}" + mode: "0755" + notify: + - Restart the GPU fan curve control service + +- name: Install the systemd unit + ansible.builtin.template: + src: gnai-gpu-fan-control.service.j2 + dest: "/etc/systemd/system/{{ gpu_fan_control_service }}.service" + mode: "0644" + notify: + - Reload the systemd daemon + - Restart the GPU fan curve control service + +- name: Enable and start the GPU fan curve control service + ansible.builtin.systemd_service: + name: "{{ gpu_fan_control_service }}" + enabled: true + state: started + when: not ansible_check_mode + +- name: Verify the GPU fan curve control service is active + ansible.builtin.command: + argv: + - /usr/bin/systemctl + - is-active + - "{{ gpu_fan_control_service }}" + register: gpu_fan_control_active + changed_when: false + failed_when: gpu_fan_control_active.stdout != 'active' + when: not ansible_check_mode + +- name: Verify the service claimed manual fan control + ansible.builtin.command: + argv: + - /bin/sh + - -c + - >- + for card in {{ gpu_fan_control_pci_addresses | join(' ') }}; + do cat /sys/bus/pci/devices/$card/hwmon/hwmon*/pwm1_enable 2>/dev/null; + done + register: gpu_fan_control_enable_state + until: gpu_fan_control_enable_state.stdout_lines | length == gpu_fan_control_pci_addresses | length + retries: 6 + delay: 2 + changed_when: false + when: not ansible_check_mode + +- name: Verify manual fan control is active on every configured card + ansible.builtin.assert: + that: + - gpu_fan_control_enable_state.stdout_lines | select('match', '^1$') | list | length == gpu_fan_control_pci_addresses | length + fail_msg: >- + The fan curve service did not claim manual control (pwm1_enable=1) + on every configured card — check the service journal. + when: not ansible_check_mode diff --git a/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.service.j2 b/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.service.j2 new file mode 100644 index 0000000..9b3bbad --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.service.j2 @@ -0,0 +1,16 @@ +[Unit] +# Fan curve rationale and measurements: GNAI-010 section 10.10 +# (infrastructure repository, docs/platform/llm-server/ai/). +Description=Ghostnet AI GPU fan curve control +After=multi-user.target + +[Service] +Type=simple +ExecStart={{ gpu_fan_control_install_dir }}/{{ gpu_fan_control_script }} +Restart=on-failure +RestartSec=5 +KillSignal=SIGTERM +TimeoutStopSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.sh.j2 b/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.sh.j2 new file mode 100644 index 0000000..0246ee2 --- /dev/null +++ b/llm-server/host/ansible/roles/gpu_fan_control/templates/gnai-gpu-fan-control.sh.j2 @@ -0,0 +1,132 @@ +#!/bin/sh +# Ghostnet AI GPU fan curve control. +# +# Drives the fan PWM of the configured amdgpu cards from their junction +# temperature, using a linear interpolation between the curve anchors. +# Rationale: the factory fan curves ramp far too slowly for the cramped +# dual-GPU layout (see GNAI-010 section 10.10). Runs as a root system +# service; the EXIT trap restores factory control (pwm1_enable=2) for every +# configured card so a stopped service never leaves a frozen PWM behind. + +set -u + +CARDS="{{ gpu_fan_control_pci_addresses | join(' ') }}" +CURVE="{% for anchor in gpu_fan_control_curve %}{{ anchor.temp_c }}:{{ anchor.pwm }}{% if not loop.last %} {% endif %}{% endfor %}" +PWM_STEP={{ gpu_fan_control_pwm_step }} +PWM_FALL_STEP={{ gpu_fan_control_pwm_step * 2 }} +INTERVAL={{ gpu_fan_control_interval_seconds }} + +log() { + echo "gnai-gpu-fan-control: $1" +} + +hwmon_dirs() { + # $1 = PCI address + ls -d /sys/bus/pci/devices/"$1"/hwmon/hwmon* 2>/dev/null +} + +control_dir() { + # $1 = PCI address; prints the first hwmon dir with a writable PWM control + for d in $(hwmon_dirs "$1"); do + if [ -w "$d/pwm1_enable" ] && [ -w "$d/pwm1" ]; then + echo "$d" + return 0 + fi + done + return 1 +} + +junction_mc() { + # $1 = PCI address; prints junction temperature in milli-celsius + for d in $(hwmon_dirs "$1"); do + i=1 + while [ -e "$d/temp${i}_label" ]; do + if [ "$(cat "$d/temp${i}_label")" = "junction" ]; then + cat "$d/temp${i}_input" + return 0 + fi + i=$((i + 1)) + done + done + return 1 +} + +target_pwm() { + # $1 = junction temperature in milli-celsius; anchors are fed one per + # line to the awk accumulator + printf '%s\n' $CURVE | awk -v t="$1" ' + { + split($0, a, ":") + n = n + 1 + T[n] = a[1] * 1000 + P[n] = a[2] + } + END { + if (t <= T[1]) { print P[1]; exit } + if (t >= T[n]) { print P[n]; exit } + for (i = 1; i < n; i++) { + if (t >= T[i] && t < T[i+1]) { + f = (t - T[i]) / (T[i+1] - T[i]) + printf "%d\n", P[i] + f * (P[i+1] - P[i]) + exit + } + } + }' +} + +restore_factory() { + for card in $CARDS; do + for d in $(hwmon_dirs "$card"); do + [ -w "$d/pwm1_enable" ] || continue + printf '2\n' > "$d/pwm1_enable" 2>/dev/null || true + log "restored factory fan control on $card" + done + done +} + +trap restore_factory EXIT INT TERM + +missing_logged=no +log "starting: cards=[$CARDS] curve='$CURVE' step=$PWM_STEP interval=${INTERVAL}s" + +while :; do + active=no + for card in $CARDS; do + ctrl=$(control_dir "$card") || continue + jmc=$(junction_mc "$card") || continue + active=yes + # Re-assert manual control on every cycle so an amdgpu reload cannot + # silently restore the factory curve mid-load. + enable=$(cat "$ctrl/pwm1_enable" 2>/dev/null) || enable="" + if [ "$enable" != "1" ]; then + if printf '1\n' > "$ctrl/pwm1_enable"; then + log "$card: claimed manual fan control" + else + continue + fi + fi + cur=$(cat "$ctrl/pwm1" 2>/dev/null) || continue + tgt=$(target_pwm "$jmc") + # Quantize the target to the step grid. + tgt=$(( (tgt / PWM_STEP) * PWM_STEP )) + # Raise immediately, lower only after falling well below the current + # value (hysteresis against oscillation). + if [ "$tgt" -gt "$cur" ] || [ "$tgt" -le $(( cur - PWM_FALL_STEP )) ]; then + if printf '%s\n' "$tgt" > "$ctrl/pwm1"; then + log "$card junction=$((jmc / 1000))C pwm=$tgt (was $cur)" + fi + fi + done + if [ "$active" = "no" ]; then + if [ "$missing_logged" = "no" ]; then + log "WARN: no writable amdgpu PWM control for configured cards" + missing_logged=yes + fi + else + missing_logged=no + fi + # Sleep in the background and wait, so signal traps are handled + # promptly instead of after the sleep finishes. + sleep "$INTERVAL" & + wait $! +done