213 lines
7.1 KiB
YAML
213 lines
7.1 KiB
YAML
---
|
|
- name: Verify AMD GPU access declarations
|
|
ansible.builtin.assert:
|
|
that:
|
|
- amd_gpu_vendor_id is match("^[0-9a-fA-F]{4}$")
|
|
- amd_gpu_device_id is match("^[0-9a-fA-F]{4}$")
|
|
- amd_gpu_expected_count | int > 0
|
|
- >-
|
|
amd_gpu_workload_accounts | map(attribute="name") | unique | length ==
|
|
amd_gpu_workload_accounts | length
|
|
- >-
|
|
amd_gpu_workload_accounts | map(attribute="uid") | unique | length ==
|
|
amd_gpu_workload_accounts | length
|
|
fail_msg: AMD GPU identifiers and workload accounts must be valid and unique.
|
|
|
|
- name: Read matching PCI devices and kernel drivers
|
|
ansible.builtin.command:
|
|
argv:
|
|
- /usr/bin/lspci
|
|
- -Dnnk
|
|
- -d
|
|
- "{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }}"
|
|
changed_when: false
|
|
check_mode: false
|
|
register: amd_gpu_lspci
|
|
|
|
- name: Verify expected GPUs use the amdgpu kernel driver
|
|
ansible.builtin.assert:
|
|
that:
|
|
- >-
|
|
amd_gpu_lspci.stdout_lines
|
|
| select("search", "(?i)" + amd_gpu_vendor_id + ":" + amd_gpu_device_id)
|
|
| list | length == amd_gpu_expected_count | int
|
|
- >-
|
|
amd_gpu_lspci.stdout
|
|
| regex_findall("Kernel driver in use: amdgpu")
|
|
| length == amd_gpu_expected_count | int
|
|
fail_msg: >-
|
|
Expected {{ amd_gpu_expected_count }} PCI devices
|
|
{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }} using amdgpu.
|
|
|
|
- name: Find DRM render nodes in sysfs
|
|
ansible.builtin.find:
|
|
paths: /sys/class/drm
|
|
patterns: "renderD*"
|
|
file_type: link
|
|
recurse: false
|
|
register: amd_gpu_sysfs_render_nodes
|
|
|
|
- name: Read render node PCI vendor IDs
|
|
ansible.builtin.slurp:
|
|
src: "{{ amd_gpu_render_node.path }}/device/vendor"
|
|
loop: "{{ amd_gpu_sysfs_render_nodes.files }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_render_node
|
|
label: "{{ amd_gpu_render_node.path | basename }}"
|
|
register: amd_gpu_render_vendor_ids
|
|
|
|
- name: Read render node PCI device IDs
|
|
ansible.builtin.slurp:
|
|
src: "{{ amd_gpu_render_node.path }}/device/device"
|
|
loop: "{{ amd_gpu_sysfs_render_nodes.files }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_render_node
|
|
label: "{{ amd_gpu_render_node.path | basename }}"
|
|
register: amd_gpu_render_device_ids
|
|
|
|
- name: Initialize matching AMD render device list
|
|
ansible.builtin.set_fact:
|
|
amd_gpu_render_devices: []
|
|
|
|
- name: Select render devices belonging to the configured AMD GPUs
|
|
ansible.builtin.set_fact:
|
|
amd_gpu_render_devices: >-
|
|
{{ amd_gpu_render_devices
|
|
+ ['/dev/dri/'
|
|
+ (amd_gpu_sysfs_render_nodes.files[amd_gpu_render_index].path
|
|
| basename)] }}
|
|
loop: "{{ range(0, amd_gpu_sysfs_render_nodes.files | length) | list }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_render_index
|
|
label: >-
|
|
{{ amd_gpu_sysfs_render_nodes.files[amd_gpu_render_index].path | basename }}
|
|
when:
|
|
- >-
|
|
amd_gpu_render_vendor_ids.results[amd_gpu_render_index].content
|
|
| b64decode | trim | lower == "0x" + amd_gpu_vendor_id | lower
|
|
- >-
|
|
amd_gpu_render_device_ids.results[amd_gpu_render_index].content
|
|
| b64decode | trim | lower == "0x" + amd_gpu_device_id | lower
|
|
|
|
- name: Verify one render device was found for each configured GPU
|
|
ansible.builtin.assert:
|
|
that:
|
|
- amd_gpu_render_devices | length == amd_gpu_expected_count
|
|
fail_msg: >-
|
|
Expected {{ amd_gpu_expected_count }} matching AMD render devices but found
|
|
{{ amd_gpu_render_devices | length }}: {{ amd_gpu_render_devices }}.
|
|
|
|
- name: Inspect the ROCm kernel device
|
|
ansible.builtin.stat:
|
|
path: /dev/kfd
|
|
register: amd_gpu_kfd
|
|
|
|
- name: Verify the ROCm kernel device exists
|
|
ansible.builtin.assert:
|
|
that:
|
|
- amd_gpu_kfd.stat.exists
|
|
- amd_gpu_kfd.stat.ischr | default(false)
|
|
- amd_gpu_kfd.stat.gr_name != "root"
|
|
fail_msg: /dev/kfd must exist as a character device with a dedicated group.
|
|
|
|
- name: Inspect matching AMD render devices
|
|
ansible.builtin.stat:
|
|
path: "{{ amd_gpu_render_device }}"
|
|
loop: "{{ amd_gpu_render_devices }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_render_device
|
|
register: amd_gpu_render_device_stats
|
|
|
|
- name: Verify matching AMD render devices are usable character devices
|
|
ansible.builtin.assert:
|
|
that:
|
|
- amd_gpu_render_stat.stat.exists
|
|
- amd_gpu_render_stat.stat.ischr | default(false)
|
|
- amd_gpu_render_stat.stat.gr_name != "root"
|
|
loop: "{{ amd_gpu_render_device_stats.results }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_render_stat
|
|
label: "{{ amd_gpu_render_stat.amd_gpu_render_device }}"
|
|
|
|
- name: Derive the minimal host groups required by the GPU devices
|
|
ansible.builtin.set_fact:
|
|
amd_gpu_required_group_names: >-
|
|
{{ ([amd_gpu_kfd.stat.gr_name]
|
|
+ (amd_gpu_render_device_stats.results
|
|
| map(attribute="stat.gr_name") | list))
|
|
| unique | sort | list }}
|
|
|
|
- name: Grant AI workload accounts access to the GPU device groups
|
|
ansible.builtin.user:
|
|
name: "{{ amd_gpu_account.name }}"
|
|
groups: "{{ amd_gpu_required_group_names | join(',') }}"
|
|
append: true
|
|
loop: "{{ amd_gpu_workload_accounts }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_account
|
|
label: "{{ amd_gpu_account.name }}"
|
|
notify: Restart AI workload user managers
|
|
|
|
- name: Apply changed group memberships to persistent user managers
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: Read effective AI workload account groups
|
|
ansible.builtin.command:
|
|
argv:
|
|
- /usr/bin/id
|
|
- -nG
|
|
- "{{ amd_gpu_account.name }}"
|
|
changed_when: false
|
|
loop: "{{ amd_gpu_workload_accounts }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_account
|
|
label: "{{ amd_gpu_account.name }}"
|
|
register: amd_gpu_account_groups
|
|
when: not ansible_check_mode
|
|
|
|
- name: Verify AI workload account group memberships
|
|
ansible.builtin.assert:
|
|
that:
|
|
- >-
|
|
amd_gpu_required_group_names
|
|
| difference(amd_gpu_account_group.stdout.split()) | length == 0
|
|
fail_msg: >-
|
|
{{ amd_gpu_account_group.amd_gpu_account.name }} is missing a required GPU
|
|
device group.
|
|
loop: "{{ amd_gpu_account_groups.results }}"
|
|
loop_control:
|
|
loop_var: amd_gpu_account_group
|
|
label: "{{ amd_gpu_account_group.amd_gpu_account.name }}"
|
|
when: not ansible_check_mode
|
|
|
|
- name: Verify AI workload accounts can read and write required GPU devices
|
|
ansible.builtin.command:
|
|
argv:
|
|
- /usr/sbin/runuser
|
|
- --user
|
|
- "{{ amd_gpu_access_pair.0.name }}"
|
|
- --
|
|
- /usr/bin/python3
|
|
- -c
|
|
- >-
|
|
import os, sys; raise SystemExit(
|
|
0 if os.access(sys.argv[1], os.R_OK | os.W_OK) else 1)
|
|
- "{{ amd_gpu_access_pair.1 }}"
|
|
changed_when: false
|
|
loop: >-
|
|
{{ amd_gpu_workload_accounts
|
|
| product(['/dev/kfd'] + amd_gpu_render_devices) | list }}
|
|
loop_control:
|
|
loop_var: amd_gpu_access_pair
|
|
label: "{{ amd_gpu_access_pair.0.name }} -> {{ amd_gpu_access_pair.1 }}"
|
|
when: not ansible_check_mode
|
|
|
|
- name: Report verified AMD GPU access
|
|
ansible.builtin.debug:
|
|
msg:
|
|
pci_id: "{{ amd_gpu_vendor_id }}:{{ amd_gpu_device_id }}"
|
|
gpu_count: "{{ amd_gpu_expected_count }}"
|
|
render_devices: "{{ amd_gpu_render_devices }}"
|
|
device_groups: "{{ amd_gpu_required_group_names }}"
|
|
accounts: "{{ amd_gpu_workload_accounts | map(attribute='name') | list }}"
|