Merge pull request #2108 from jeburks2/mig-overlay
Nvidia MIG Configuration via new MIG Overlay
This commit is contained in:
Christian Goll
GitHub
@@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||
|
||||
- New --partwipe flag for profile and node set
|
||||
- Updated arguments for `ValidString` to match `regexp.MatchString`
|
||||
- New `mig` overlay to configure NVIDIA MIG devices. #2102
|
||||
- Updated arguments for `ValidString` to match `regexp.MatchString`
|
||||
- New `mig` overlay to configure NVIDIA MIG devices. #2102
|
||||
|
||||
## v4.6.5, 2026-01-12
|
||||
|
||||
|
||||
226
overlays/mig/internal/mig_test.go
Normal file
226
overlays/mig/internal/mig_test.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package mig
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/warewulf/warewulf/internal/app/wwctl/overlay/info"
|
||||
"github.com/warewulf/warewulf/internal/app/wwctl/overlay/show"
|
||||
"github.com/warewulf/warewulf/internal/pkg/testenv"
|
||||
"github.com/warewulf/warewulf/internal/pkg/wwlog"
|
||||
)
|
||||
|
||||
func Test_migOverlay(t *testing.T) {
|
||||
env := testenv.New(t)
|
||||
defer env.RemoveAll()
|
||||
env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf")
|
||||
env.ImportFile("var/lib/warewulf/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww", "../rootfs/etc/systemd/system/ww-nvidia-mig.service.ww")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
log string
|
||||
}{
|
||||
{
|
||||
name: "mig:nvidia-mig.service (default)",
|
||||
args: []string{"--render", "node1", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"},
|
||||
log: migServiceDefault,
|
||||
},
|
||||
{
|
||||
name: "mig:nvidia-mig.service (homogeneous)",
|
||||
args: []string{"--render", "node2", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"},
|
||||
log: migServiceHomogeneous,
|
||||
},
|
||||
{
|
||||
name: "mig:nvidia-mig.service (heterogeneous)",
|
||||
args: []string{"--render", "node3", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"},
|
||||
log: migServiceHeterogeneous,
|
||||
},
|
||||
{
|
||||
name: "mig:nvidia-mig.service (heterogeneous short names)",
|
||||
args: []string{"--render", "node4", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"},
|
||||
log: migServiceHeterogeneousShortNames,
|
||||
},
|
||||
}
|
||||
infoTests := []struct {
|
||||
name string
|
||||
args []string
|
||||
log string
|
||||
stdout string
|
||||
}{
|
||||
{
|
||||
name: "mig:nvidia-mig.service (info)",
|
||||
args: []string{"mig", "etc/systemd/system/ww-nvidia-mig.service.ww"},
|
||||
log: migInfoOutput,
|
||||
stdout: migInfoStdout,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cmd := show.GetCommand()
|
||||
cmd.SetArgs(tt.args)
|
||||
stdout := bytes.NewBufferString("")
|
||||
stderr := bytes.NewBufferString("")
|
||||
logbuf := bytes.NewBufferString("")
|
||||
cmd.SetOut(stdout)
|
||||
cmd.SetErr(stderr)
|
||||
wwlog.SetLogWriter(logbuf)
|
||||
err := cmd.Execute()
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, stdout.String())
|
||||
assert.Empty(t, stderr.String())
|
||||
assert.Equal(t, tt.log, logbuf.String())
|
||||
})
|
||||
}
|
||||
|
||||
for _, tt := range infoTests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cmd := info.GetCommand()
|
||||
cmd.SetArgs(tt.args)
|
||||
stdout := bytes.NewBufferString("")
|
||||
stderr := bytes.NewBufferString("")
|
||||
logbuf := bytes.NewBufferString("")
|
||||
cmd.SetOut(stdout)
|
||||
cmd.SetErr(stderr)
|
||||
wwlog.SetLogWriter(logbuf)
|
||||
err := cmd.Execute()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.log, logbuf.String())
|
||||
assert.Equal(t, tt.stdout, stdout.String())
|
||||
assert.Empty(t, stderr.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const migServiceDefault string = `backupFile: true
|
||||
writeFile: true
|
||||
Filename: etc/systemd/system/ww-nvidia-mig.service
|
||||
# This file is autogenerated by warewulf
|
||||
|
||||
[Unit]
|
||||
DefaultDependencies=no
|
||||
Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf
|
||||
Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=-/usr/bin/nvidia-smi -pm 1
|
||||
ExecStartPre=/usr/bin/nvidia-smi -mig 1
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 2,2 -C
|
||||
ExecStartPost=-/usr/bin/nvidia-smi -L
|
||||
ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dci
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dgi
|
||||
ExecStop=/usr/bin/nvidia-smi -mig 0
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const migServiceHomogeneous string = `backupFile: true
|
||||
writeFile: true
|
||||
Filename: etc/systemd/system/ww-nvidia-mig.service
|
||||
# This file is autogenerated by warewulf
|
||||
|
||||
[Unit]
|
||||
DefaultDependencies=no
|
||||
Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf
|
||||
Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=-/usr/bin/nvidia-smi -pm 1
|
||||
ExecStartPre=/usr/bin/nvidia-smi -mig 1
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -C
|
||||
ExecStartPost=-/usr/bin/nvidia-smi -L
|
||||
ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dci
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dgi
|
||||
ExecStop=/usr/bin/nvidia-smi -mig 0
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const migServiceHeterogeneous string = `backupFile: true
|
||||
writeFile: true
|
||||
Filename: etc/systemd/system/ww-nvidia-mig.service
|
||||
# This file is autogenerated by warewulf
|
||||
|
||||
[Unit]
|
||||
DefaultDependencies=no
|
||||
Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf
|
||||
Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=-/usr/bin/nvidia-smi -pm 1
|
||||
ExecStartPre=/usr/bin/nvidia-smi -mig 1
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -i 0 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 9,9 -i 1 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C
|
||||
ExecStartPost=-/usr/bin/nvidia-smi -L
|
||||
ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dci
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dgi
|
||||
ExecStop=/usr/bin/nvidia-smi -mig 0
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const migServiceHeterogeneousShortNames string = `backupFile: true
|
||||
writeFile: true
|
||||
Filename: etc/systemd/system/ww-nvidia-mig.service
|
||||
# This file is autogenerated by warewulf
|
||||
|
||||
[Unit]
|
||||
DefaultDependencies=no
|
||||
Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf
|
||||
Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=-/usr/bin/nvidia-smi -pm 1
|
||||
ExecStartPre=/usr/bin/nvidia-smi -mig 1
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb -i 0 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 3g.90gb,3g.90gb -i 1 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C
|
||||
ExecStartPost=-/usr/bin/nvidia-smi -L
|
||||
ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dci
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dgi
|
||||
ExecStop=/usr/bin/nvidia-smi -mig 0
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
|
||||
const migInfoOutput string = `Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node.
|
||||
`
|
||||
|
||||
const migInfoStdout string = `
|
||||
VARIABLE OPTION TYPE HELP
|
||||
-------- ------ ---- ----
|
||||
.Tags.gpuMigProfiles string List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index
|
||||
`
|
||||
11
overlays/mig/internal/nodes.conf
Normal file
11
overlays/mig/internal/nodes.conf
Normal file
@@ -0,0 +1,11 @@
|
||||
nodes:
|
||||
node1: {}
|
||||
node2:
|
||||
tags:
|
||||
gpuMigProfiles: "14,14,14,15"
|
||||
node3:
|
||||
tags:
|
||||
gpuMigProfiles: "14,14,14,15:9,9:0:0"
|
||||
node4:
|
||||
tags:
|
||||
gpuMigProfiles: "1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb:3g.90gb,3g.90gb:0:0"
|
||||
@@ -0,0 +1 @@
|
||||
../nvidia-mig.service
|
||||
@@ -0,0 +1,37 @@
|
||||
# This file is autogenerated by warewulf
|
||||
|
||||
{{- /* wwdoc: Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node. */}}
|
||||
{{- /* .Tags.gpuMigProfiles: List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index */}}
|
||||
|
||||
[Unit]
|
||||
DefaultDependencies=no
|
||||
Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf
|
||||
Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service
|
||||
ConditionPathExists=/usr/bin/nvidia-smi
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=-/usr/bin/nvidia-smi -pm 1
|
||||
ExecStartPre=/usr/bin/nvidia-smi -mig 1
|
||||
{{- $rawMigProfiles := default "2,2" .Tags.gpuMigProfiles -}}
|
||||
{{- $migProfiles := splitList ":" $rawMigProfiles -}}
|
||||
{{- if gt (len $migProfiles) 1 -}}
|
||||
{{- range $gpuIndex, $migProfile := $migProfiles -}}
|
||||
{{- if ne $migProfile "" }}
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi {{$migProfile}} -i {{$gpuIndex}} -C
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- else }}
|
||||
ExecStart=/usr/bin/nvidia-smi mig -cgi {{$rawMigProfiles}} -C
|
||||
{{- end }}
|
||||
ExecStartPost=-/usr/bin/nvidia-smi -L
|
||||
ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dci
|
||||
ExecStop=-/usr/bin/nvidia-smi mig -dgi
|
||||
ExecStop=/usr/bin/nvidia-smi -mig 0
|
||||
TimeoutStartSec=300
|
||||
TimeoutStopSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
46
overlays/mig/rootfs/usr/local/sbin/mig2gres
Normal file
46
overlays/mig/rootfs/usr/local/sbin/mig2gres
Normal file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Generates NVIDIA MIG device entries for Slurm's gres.conf.
|
||||
# Queries MIG GPU instances and maps them to /dev/nvidia-caps/ device files.
|
||||
#
|
||||
# Process: nvidia-smi -> extract GPU/GI/profile -> lookup minor in
|
||||
# /proc/driver/nvidia-caps/mig-minors -> map to device file -> output gres format
|
||||
|
||||
|
||||
if ! command -v nvidia-smi &> /dev/null; then
|
||||
echo "Error: nvidia-smi command not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v grep &> /dev/null; then
|
||||
echo "Error: grep command not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v awk &> /dev/null; then
|
||||
echo "Error: awk command not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v hostname &> /dev/null; then
|
||||
echo "Error: hostname command not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
hostname=$(hostname -s)
|
||||
|
||||
nvidia-smi mig -lgi | grep -E '^\|' | grep -v '===' | grep -v 'GPU.*Name' | while read line; do
|
||||
gpu=$(echo "$line" | awk '{print $2}')
|
||||
name=$(echo "$line" | awk '{print $4}')
|
||||
gi=$(echo "$line" | awk '{print $6}') # Instance ID is column 6
|
||||
|
||||
[[ "$gpu" =~ ^[0-9]+$ ]] || continue
|
||||
|
||||
pattern="gpu${gpu}/gi${gi}/access"
|
||||
minor=$(grep "^${pattern} " /proc/driver/nvidia-caps/mig-minors | awk '{print $2}')
|
||||
device_file="/dev/nvidia-caps/nvidia-cap${minor}"
|
||||
|
||||
echo "NodeName=${hostname} Name=gpu Type=${name} File=${device_file}"
|
||||
done
|
||||
@@ -527,6 +527,70 @@ Two overlays, **systemd.mount** and **systemd.swap**, configure mounted and swap
|
||||
storage based on the configuration of native file system fields. They are often
|
||||
paired with the ``mkfs`` and ``mkswap`` overlays.
|
||||
|
||||
mig
|
||||
----
|
||||
|
||||
The **mig** overlay configures NVIDIA MIG (Multi-Instance GPU) settings on nodes
|
||||
with supported NVIDIA GPUs. The **mig** overlay requires the NVIDIA driver and
|
||||
the ``nvidia-smi`` tool to be installed and functional on the node.
|
||||
|
||||
MIG configuration is based on the ``gpuMigProfiles`` tag, which specifies a
|
||||
comma-separated list of MIG profiles IDs or profile short names to configure.
|
||||
|
||||
For example, to configure a node with four MIG instances of profile 14 and one
|
||||
instance of profile 15 on all GPUs:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15"
|
||||
|
||||
The MIG instances can also be defined by their profile short names:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb"
|
||||
|
||||
If different GPUs on a node require different MIG configurations, the
|
||||
``gpuMigProfiles`` tag can be specified as a colon-separated list of comma-separated
|
||||
lists. Commas separate profiles on a single GPU. Colons separate GPUs by index order.
|
||||
A value of 0 indicates no MIG partitioning and uses the full GPU.
|
||||
|
||||
For example, to configure a node that has four GPUs, where:
|
||||
|
||||
- GPU 0 has three instances of profile 14 and one instance of profile 15
|
||||
- GPU 1 has two instances of profile 9
|
||||
- GPU 2 and GPU 3 have one instance of profile 0 (no partitioning):
|
||||
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15:9,9:0:0"
|
||||
|
||||
This can also be done using profile short names:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb:3g.40gb,3g.40gb:0:0"
|
||||
|
||||
The exact MIG configuration available depends on the specific NVIDIA GPU model
|
||||
present in the node. To see the available MIG profiles for a given GPU model, run:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
nvidia-smi mig -lgip
|
||||
|
||||
Refer to `NVIDIA’s documentation <https://docs.nvidia.com/datacenter/tesla/mig-user-guide/supported-mig-profiles.html>`_ for more information on MIG profiles.
|
||||
|
||||
Once configured, the NVIDIA MIG instances can be mapped to Slurm GRES entries
|
||||
backed by ``/dev/nvidia-caps/capXX`` device files.
|
||||
|
||||
To generate the mapping of MIG instances to ``/dev/nvidia-caps/capXX`` device files
|
||||
for Slurm's ``gres.conf`` file, use the script provided by this overlay on on the node:
|
||||
|
||||
.. code-block:: shell
|
||||
|
||||
/usr/bin/sh /usr/local/sbin/mig2gres
|
||||
|
||||
host
|
||||
----
|
||||
|
||||
|
||||
@@ -224,6 +224,7 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup}
|
||||
%{_overlaydir}/mkswap/rootfs/*
|
||||
%{_overlaydir}/systemd.mount/rootfs/*
|
||||
%{_overlaydir}/systemd.swap/rootfs/*
|
||||
%{_overlaydir}/mig/rootfs/*
|
||||
|
||||
%{_bindir}/wwctl
|
||||
%{_prefix}/lib/firewalld/services/warewulf.xml
|
||||
|
||||
Reference in New Issue
Block a user