GNAI007: Render- und GPU-Rechte den beiden KI-Konten hinzugefügt.
This commit is contained in:
+24
-4
@@ -2,8 +2,9 @@
|
||||
|
||||
This directory contains the reproducible control project for `ghostnet-ki`.
|
||||
The read-only `playbooks/preflight.yml` verifies the documented host identity
|
||||
and hardware. The first writing milestone, `playbooks/host.yml`, applies a
|
||||
small shared host baseline through the dedicated deployment account.
|
||||
and hardware. The shared `playbooks/host.yml` applies the host baseline,
|
||||
Podman runtime, workload identities and narrowly scoped AMD device access
|
||||
through the dedicated deployment account.
|
||||
|
||||
## Control-node setup
|
||||
|
||||
@@ -166,5 +167,24 @@ UID/GID and subordinate-ID ranges:
|
||||
Each account has a persistent systemd user manager and a separate,
|
||||
administrator-managed directory below `/etc/containers/systemd/users/<UID>`.
|
||||
Configuration is root-owned and group-readable; workload data belongs to its
|
||||
service account. The role does not grant GPU access and creates no workload
|
||||
containers.
|
||||
service account. The two AI identities have `render` as their explicit
|
||||
supplementary-group baseline; the encoding identity has no GPU group. The role
|
||||
creates no workload containers.
|
||||
|
||||
## AMD GPU access
|
||||
|
||||
The `amd_gpu_access` role verifies exactly two PCI devices `1002:73bf` using
|
||||
the `amdgpu` kernel driver. It maps their sysfs identities to the corresponding
|
||||
DRM render nodes, checks `/dev/kfd`, derives the device groups and confirms
|
||||
read/write access as both AI workload identities.
|
||||
|
||||
On the current host, the verified devices are `/dev/kfd`,
|
||||
`/dev/dri/renderD128` and `/dev/dri/renderD129`; all use group `render`.
|
||||
`gnai-production` and `gnai-testing` can access them, while `gn-encoding`
|
||||
remains outside that group. The final convergence run reported
|
||||
`ok=55 changed=0 unreachable=0 failed=0`.
|
||||
|
||||
This proves host-level device permissions only. No ROCm userspace image has
|
||||
been pulled and no GPU workload has run yet. The next workload check must pass
|
||||
these devices into a rootless container with `GroupAdd=keep-groups` and prove
|
||||
that `llama.cpp` detects both cards as `gfx1030`.
|
||||
|
||||
@@ -12,3 +12,4 @@
|
||||
- role: host_baseline
|
||||
- role: podman_runtime
|
||||
- role: workload_accounts
|
||||
- role: amd_gpu_access
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# AMD GPU access role
|
||||
|
||||
This shared host role verifies the two configured AMD GPUs and their `amdgpu`
|
||||
kernel driver, discovers only the DRM render nodes belonging to their PCI IDs,
|
||||
and checks `/dev/kfd`. It grants the Production and Testing service accounts
|
||||
the minimal supplementary groups required by those device nodes.
|
||||
|
||||
When group membership changes, the role restarts both persistent systemd user
|
||||
managers so later rootless Quadlets can use `GroupAdd=keep-groups`. It verifies
|
||||
that each service account can read and write `/dev/kfd` and both selected
|
||||
render devices.
|
||||
|
||||
The role installs no ROCm userspace packages, pulls no image and starts no
|
||||
container. ROCm userspace will come from the pinned `llama.cpp` container. A
|
||||
later workload check must still prove that the container detects both GPUs as
|
||||
`gfx1030` and performs inference on them.
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
amd_gpu_vendor_id: "1002"
|
||||
amd_gpu_device_id: "73bf"
|
||||
amd_gpu_expected_count: 2
|
||||
|
||||
amd_gpu_workload_accounts:
|
||||
- name: gnai-production
|
||||
uid: 980
|
||||
- name: gnai-testing
|
||||
uid: 981
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Restart AI workload user managers
|
||||
ansible.builtin.systemd_service:
|
||||
name: "user@{{ amd_gpu_account.uid }}.service"
|
||||
state: restarted
|
||||
loop: "{{ amd_gpu_workload_accounts }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_account
|
||||
label: "{{ amd_gpu_account.name }}"
|
||||
when: not ansible_check_mode
|
||||
@@ -0,0 +1,212 @@
|
||||
---
|
||||
- name: Verify AMD GPU access declarations
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- amd_gpu_vendor_id is match("^[0-9a-fA-F]{4}$")
|
||||
- amd_gpu_device_id is match("^[0-9a-fA-F]{4}$")
|
||||
- amd_gpu_expected_count | int > 0
|
||||
- >-
|
||||
amd_gpu_workload_accounts | map(attribute="name") | unique | length ==
|
||||
amd_gpu_workload_accounts | length
|
||||
- >-
|
||||
amd_gpu_workload_accounts | map(attribute="uid") | unique | length ==
|
||||
amd_gpu_workload_accounts | length
|
||||
fail_msg: AMD GPU identifiers and workload accounts must be valid and unique.
|
||||
|
||||
- name: Read matching PCI devices and kernel drivers
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/bin/lspci
|
||||
- -Dnnk
|
||||
- -d
|
||||
- "{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }}"
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
register: amd_gpu_lspci
|
||||
|
||||
- name: Verify expected GPUs use the amdgpu kernel driver
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
amd_gpu_lspci.stdout_lines
|
||||
| select("search", "(?i)" + amd_gpu_vendor_id + ":" + amd_gpu_device_id)
|
||||
| list | length == amd_gpu_expected_count | int
|
||||
- >-
|
||||
amd_gpu_lspci.stdout
|
||||
| regex_findall("Kernel driver in use: amdgpu")
|
||||
| length == amd_gpu_expected_count | int
|
||||
fail_msg: >-
|
||||
Expected {{ amd_gpu_expected_count }} PCI devices
|
||||
{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }} using amdgpu.
|
||||
|
||||
- name: Find DRM render nodes in sysfs
|
||||
ansible.builtin.find:
|
||||
paths: /sys/class/drm
|
||||
patterns: "renderD*"
|
||||
file_type: link
|
||||
recurse: false
|
||||
register: amd_gpu_sysfs_render_nodes
|
||||
|
||||
- name: Read render node PCI vendor IDs
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ amd_gpu_render_node.path }}/device/vendor"
|
||||
loop: "{{ amd_gpu_sysfs_render_nodes.files }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_render_node
|
||||
label: "{{ amd_gpu_render_node.path | basename }}"
|
||||
register: amd_gpu_render_vendor_ids
|
||||
|
||||
- name: Read render node PCI device IDs
|
||||
ansible.builtin.slurp:
|
||||
src: "{{ amd_gpu_render_node.path }}/device/device"
|
||||
loop: "{{ amd_gpu_sysfs_render_nodes.files }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_render_node
|
||||
label: "{{ amd_gpu_render_node.path | basename }}"
|
||||
register: amd_gpu_render_device_ids
|
||||
|
||||
- name: Initialize matching AMD render device list
|
||||
ansible.builtin.set_fact:
|
||||
amd_gpu_render_devices: []
|
||||
|
||||
- name: Select render devices belonging to the configured AMD GPUs
|
||||
ansible.builtin.set_fact:
|
||||
amd_gpu_render_devices: >-
|
||||
{{ amd_gpu_render_devices
|
||||
+ ['/dev/dri/'
|
||||
+ (amd_gpu_sysfs_render_nodes.files[amd_gpu_render_index].path
|
||||
| basename)] }}
|
||||
loop: "{{ range(0, amd_gpu_sysfs_render_nodes.files | length) | list }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_render_index
|
||||
label: >-
|
||||
{{ amd_gpu_sysfs_render_nodes.files[amd_gpu_render_index].path | basename }}
|
||||
when:
|
||||
- >-
|
||||
amd_gpu_render_vendor_ids.results[amd_gpu_render_index].content
|
||||
| b64decode | trim | lower == "0x" + amd_gpu_vendor_id | lower
|
||||
- >-
|
||||
amd_gpu_render_device_ids.results[amd_gpu_render_index].content
|
||||
| b64decode | trim | lower == "0x" + amd_gpu_device_id | lower
|
||||
|
||||
- name: Verify one render device was found for each configured GPU
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- amd_gpu_render_devices | length == amd_gpu_expected_count
|
||||
fail_msg: >-
|
||||
Expected {{ amd_gpu_expected_count }} matching AMD render devices but found
|
||||
{{ amd_gpu_render_devices | length }}: {{ amd_gpu_render_devices }}.
|
||||
|
||||
- name: Inspect the ROCm kernel device
|
||||
ansible.builtin.stat:
|
||||
path: /dev/kfd
|
||||
register: amd_gpu_kfd
|
||||
|
||||
- name: Verify the ROCm kernel device exists
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- amd_gpu_kfd.stat.exists
|
||||
- amd_gpu_kfd.stat.ischr | default(false)
|
||||
- amd_gpu_kfd.stat.gr_name != "root"
|
||||
fail_msg: /dev/kfd must exist as a character device with a dedicated group.
|
||||
|
||||
- name: Inspect matching AMD render devices
|
||||
ansible.builtin.stat:
|
||||
path: "{{ amd_gpu_render_device }}"
|
||||
loop: "{{ amd_gpu_render_devices }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_render_device
|
||||
register: amd_gpu_render_device_stats
|
||||
|
||||
- name: Verify matching AMD render devices are usable character devices
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- amd_gpu_render_stat.stat.exists
|
||||
- amd_gpu_render_stat.stat.ischr | default(false)
|
||||
- amd_gpu_render_stat.stat.gr_name != "root"
|
||||
loop: "{{ amd_gpu_render_device_stats.results }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_render_stat
|
||||
label: "{{ amd_gpu_render_stat.amd_gpu_render_device }}"
|
||||
|
||||
- name: Derive the minimal host groups required by the GPU devices
|
||||
ansible.builtin.set_fact:
|
||||
amd_gpu_required_group_names: >-
|
||||
{{ ([amd_gpu_kfd.stat.gr_name]
|
||||
+ (amd_gpu_render_device_stats.results
|
||||
| map(attribute="stat.gr_name") | list))
|
||||
| unique | sort | list }}
|
||||
|
||||
- name: Grant AI workload accounts access to the GPU device groups
|
||||
ansible.builtin.user:
|
||||
name: "{{ amd_gpu_account.name }}"
|
||||
groups: "{{ amd_gpu_required_group_names | join(',') }}"
|
||||
append: true
|
||||
loop: "{{ amd_gpu_workload_accounts }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_account
|
||||
label: "{{ amd_gpu_account.name }}"
|
||||
notify: Restart AI workload user managers
|
||||
|
||||
- name: Apply changed group memberships to persistent user managers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Read effective AI workload account groups
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/bin/id
|
||||
- -nG
|
||||
- "{{ amd_gpu_account.name }}"
|
||||
changed_when: false
|
||||
loop: "{{ amd_gpu_workload_accounts }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_account
|
||||
label: "{{ amd_gpu_account.name }}"
|
||||
register: amd_gpu_account_groups
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Verify AI workload account group memberships
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- >-
|
||||
amd_gpu_required_group_names
|
||||
| difference(amd_gpu_account_group.stdout.split()) | length == 0
|
||||
fail_msg: >-
|
||||
{{ amd_gpu_account_group.amd_gpu_account.name }} is missing a required GPU
|
||||
device group.
|
||||
loop: "{{ amd_gpu_account_groups.results }}"
|
||||
loop_control:
|
||||
loop_var: amd_gpu_account_group
|
||||
label: "{{ amd_gpu_account_group.amd_gpu_account.name }}"
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Verify AI workload accounts can read and write required GPU devices
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/sbin/runuser
|
||||
- --user
|
||||
- "{{ amd_gpu_access_pair.0.name }}"
|
||||
- --
|
||||
- /usr/bin/python3
|
||||
- -c
|
||||
- >-
|
||||
import os, sys; raise SystemExit(
|
||||
0 if os.access(sys.argv[1], os.R_OK | os.W_OK) else 1)
|
||||
- "{{ amd_gpu_access_pair.1 }}"
|
||||
changed_when: false
|
||||
loop: >-
|
||||
{{ amd_gpu_workload_accounts
|
||||
| product(['/dev/kfd'] + amd_gpu_render_devices) | list }}
|
||||
loop_control:
|
||||
loop_var: amd_gpu_access_pair
|
||||
label: "{{ amd_gpu_access_pair.0.name }} -> {{ amd_gpu_access_pair.1 }}"
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: Report verified AMD GPU access
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
pci_id: "{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }}"
|
||||
gpu_count: "{{ amd_gpu_expected_count }}"
|
||||
render_devices: "{{ amd_gpu_render_devices }}"
|
||||
device_groups: "{{ amd_gpu_required_group_names }}"
|
||||
accounts: "{{ amd_gpu_workload_accounts | map(attribute='name') | list }}"
|
||||
@@ -11,6 +11,7 @@ service group. Workload data and the rootless Podman store belong to the
|
||||
service account. The role verifies that all three Podman stores are rootless
|
||||
and use distinct graph roots.
|
||||
|
||||
The role does not grant GPU device access and does not define containers,
|
||||
networks, schedules or resource limits. Those states belong to the AI and
|
||||
video-encoding workload roles.
|
||||
The two AI accounts receive `render` as an explicit supplementary-group
|
||||
baseline; the encoding account receives no GPU group. Hardware discovery and
|
||||
effective device-access checks belong to `amd_gpu_access`. This role does not
|
||||
define containers, networks, schedules or resource limits.
|
||||
|
||||
@@ -7,7 +7,8 @@ workload_accounts:
|
||||
home: /var/lib/ghostnet-ai/production
|
||||
config_directory: /etc/ghostnet-ai/production
|
||||
subid_start: 362144
|
||||
supplementary_groups: []
|
||||
supplementary_groups:
|
||||
- render
|
||||
- name: gnai-testing
|
||||
uid: 981
|
||||
gid: 981
|
||||
@@ -15,7 +16,8 @@ workload_accounts:
|
||||
home: /var/lib/ghostnet-ai/testing
|
||||
config_directory: /etc/ghostnet-ai/testing
|
||||
subid_start: 427680
|
||||
supplementary_groups: []
|
||||
supplementary_groups:
|
||||
- render
|
||||
- name: gn-encoding
|
||||
uid: 982
|
||||
gid: 982
|
||||
|
||||
Reference in New Issue
Block a user