diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f339c3..f6030044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - New --partwipe flag for profile and node set - Updated arguments for `ValidString` to match `regexp.MatchString` +- New `mig` overlay to configure NVIDIA MIG devices. #2102 +- Updated arguments for `ValidString` to match `regexp.MatchString` +- New `mig` overlay to configure NVIDIA MIG devices. #2102 ## v4.6.5, 2026-01-12 diff --git a/overlays/mig/internal/mig_test.go b/overlays/mig/internal/mig_test.go new file mode 100644 index 00000000..1ba7a0ac --- /dev/null +++ b/overlays/mig/internal/mig_test.go @@ -0,0 +1,226 @@ +package mig + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/warewulf/warewulf/internal/app/wwctl/overlay/info" + "github.com/warewulf/warewulf/internal/app/wwctl/overlay/show" + "github.com/warewulf/warewulf/internal/pkg/testenv" + "github.com/warewulf/warewulf/internal/pkg/wwlog" +) + +func Test_migOverlay(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll() + env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf") + env.ImportFile("var/lib/warewulf/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww", "../rootfs/etc/systemd/system/ww-nvidia-mig.service.ww") + + tests := []struct { + name string + args []string + log string + }{ + { + name: "mig:nvidia-mig.service (default)", + args: []string{"--render", "node1", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceDefault, + }, + { + name: "mig:nvidia-mig.service (homogeneous)", + args: []string{"--render", "node2", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHomogeneous, + }, + { + name: "mig:nvidia-mig.service (heterogeneous)", + args: []string{"--render", "node3", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHeterogeneous, + }, + { + name: "mig:nvidia-mig.service (heterogeneous short names)", + args: []string{"--render", "node4", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHeterogeneousShortNames, + }, + } + infoTests := []struct { + name string + args []string + log string + stdout string + }{ + { + name: "mig:nvidia-mig.service (info)", + args: []string{"mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migInfoOutput, + stdout: migInfoStdout, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := show.GetCommand() + cmd.SetArgs(tt.args) + stdout := bytes.NewBufferString("") + stderr := bytes.NewBufferString("") + logbuf := bytes.NewBufferString("") + cmd.SetOut(stdout) + cmd.SetErr(stderr) + wwlog.SetLogWriter(logbuf) + err := cmd.Execute() + assert.NoError(t, err) + assert.Empty(t, stdout.String()) + assert.Empty(t, stderr.String()) + assert.Equal(t, tt.log, logbuf.String()) + }) + } + + for _, tt := range infoTests { + t.Run(tt.name, func(t *testing.T) { + cmd := info.GetCommand() + cmd.SetArgs(tt.args) + stdout := bytes.NewBufferString("") + stderr := bytes.NewBufferString("") + logbuf := bytes.NewBufferString("") + cmd.SetOut(stdout) + cmd.SetErr(stderr) + wwlog.SetLogWriter(logbuf) + err := cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, tt.log, logbuf.String()) + assert.Equal(t, tt.stdout, stdout.String()) + assert.Empty(t, stderr.String()) + }) + } +} + +const migServiceDefault string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 2,2 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target +` + +const migServiceHomogeneous string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target +` + +const migServiceHeterogeneous string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -i 0 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 9,9 -i 1 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target +` + +const migServiceHeterogeneousShortNames string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb -i 0 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 3g.90gb,3g.90gb -i 1 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target +` + +const migInfoOutput string = `Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node. +` + +const migInfoStdout string = ` +VARIABLE OPTION TYPE HELP +-------- ------ ---- ---- +.Tags.gpuMigProfiles string List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index +` diff --git a/overlays/mig/internal/nodes.conf b/overlays/mig/internal/nodes.conf new file mode 100644 index 00000000..88564347 --- /dev/null +++ b/overlays/mig/internal/nodes.conf @@ -0,0 +1,11 @@ +nodes: + node1: {} + node2: + tags: + gpuMigProfiles: "14,14,14,15" + node3: + tags: + gpuMigProfiles: "14,14,14,15:9,9:0:0" + node4: + tags: + gpuMigProfiles: "1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb:3g.90gb,3g.90gb:0:0" \ No newline at end of file diff --git a/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/ww-nvidia-mig.service b/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/ww-nvidia-mig.service new file mode 120000 index 00000000..fc15c38c --- /dev/null +++ b/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/ww-nvidia-mig.service @@ -0,0 +1 @@ +../nvidia-mig.service \ No newline at end of file diff --git a/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww new file mode 100644 index 00000000..34d89e30 --- /dev/null +++ b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww @@ -0,0 +1,37 @@ +# This file is autogenerated by warewulf + +{{- /* wwdoc: Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node. */}} +{{- /* .Tags.gpuMigProfiles: List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index */}} + +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +{{- $rawMigProfiles := default "2,2" .Tags.gpuMigProfiles -}} +{{- $migProfiles := splitList ":" $rawMigProfiles -}} +{{- if gt (len $migProfiles) 1 -}} +{{- range $gpuIndex, $migProfile := $migProfiles -}} + {{- if ne $migProfile "" }} +ExecStart=/usr/bin/nvidia-smi mig -cgi {{$migProfile}} -i {{$gpuIndex}} -C + {{- end -}} +{{- end -}} +{{- else }} +ExecStart=/usr/bin/nvidia-smi mig -cgi {{$rawMigProfiles}} -C +{{- end }} +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/overlays/mig/rootfs/usr/local/sbin/mig2gres b/overlays/mig/rootfs/usr/local/sbin/mig2gres new file mode 100644 index 00000000..abd389cd --- /dev/null +++ b/overlays/mig/rootfs/usr/local/sbin/mig2gres @@ -0,0 +1,46 @@ +#!/usr/bin/bash + +set -euo pipefail + +# Generates NVIDIA MIG device entries for Slurm's gres.conf. +# Queries MIG GPU instances and maps them to /dev/nvidia-caps/ device files. +# +# Process: nvidia-smi -> extract GPU/GI/profile -> lookup minor in +# /proc/driver/nvidia-caps/mig-minors -> map to device file -> output gres format + + +if ! command -v nvidia-smi &> /dev/null; then + echo "Error: nvidia-smi command not found" >&2 + exit 1 +fi + +if ! command -v grep &> /dev/null; then + echo "Error: grep command not found" >&2 + exit 1 +fi + +if ! command -v awk &> /dev/null; then + echo "Error: awk command not found" >&2 + exit 1 +fi + +if ! command -v hostname &> /dev/null; then + echo "Error: hostname command not found" >&2 + exit 1 +fi + +hostname=$(hostname -s) + +nvidia-smi mig -lgi | grep -E '^\|' | grep -v '===' | grep -v 'GPU.*Name' | while read line; do + gpu=$(echo "$line" | awk '{print $2}') + name=$(echo "$line" | awk '{print $4}') + gi=$(echo "$line" | awk '{print $6}') # Instance ID is column 6 + + [[ "$gpu" =~ ^[0-9]+$ ]] || continue + + pattern="gpu${gpu}/gi${gi}/access" + minor=$(grep "^${pattern} " /proc/driver/nvidia-caps/mig-minors | awk '{print $2}') + device_file="/dev/nvidia-caps/nvidia-cap${minor}" + + echo "NodeName=${hostname} Name=gpu Type=${name} File=${device_file}" +done \ No newline at end of file diff --git a/userdocs/overlays/overlays.rst b/userdocs/overlays/overlays.rst index 6e070fec..23bca8e7 100644 --- a/userdocs/overlays/overlays.rst +++ b/userdocs/overlays/overlays.rst @@ -527,6 +527,70 @@ Two overlays, **systemd.mount** and **systemd.swap**, configure mounted and swap storage based on the configuration of native file system fields. They are often paired with the ``mkfs`` and ``mkswap`` overlays. +mig +---- + +The **mig** overlay configures NVIDIA MIG (Multi-Instance GPU) settings on nodes +with supported NVIDIA GPUs. The **mig** overlay requires the NVIDIA driver and +the ``nvidia-smi`` tool to be installed and functional on the node. + +MIG configuration is based on the ``gpuMigProfiles`` tag, which specifies a +comma-separated list of MIG profiles IDs or profile short names to configure. + +For example, to configure a node with four MIG instances of profile 14 and one +instance of profile 15 on all GPUs: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15" + +The MIG instances can also be defined by their profile short names: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb" + +If different GPUs on a node require different MIG configurations, the +``gpuMigProfiles`` tag can be specified as a colon-separated list of comma-separated +lists. Commas separate profiles on a single GPU. Colons separate GPUs by index order. +A value of 0 indicates no MIG partitioning and uses the full GPU. + +For example, to configure a node that has four GPUs, where: + +- GPU 0 has three instances of profile 14 and one instance of profile 15 +- GPU 1 has two instances of profile 9 +- GPU 2 and GPU 3 have one instance of profile 0 (no partitioning): + + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15:9,9:0:0" + +This can also be done using profile short names: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb:3g.40gb,3g.40gb:0:0" + +The exact MIG configuration available depends on the specific NVIDIA GPU model +present in the node. To see the available MIG profiles for a given GPU model, run: + +.. code-block:: shell + + nvidia-smi mig -lgip + +Refer to `NVIDIA’s documentation `_ for more information on MIG profiles. + +Once configured, the NVIDIA MIG instances can be mapped to Slurm GRES entries +backed by ``/dev/nvidia-caps/capXX`` device files. + +To generate the mapping of MIG instances to ``/dev/nvidia-caps/capXX`` device files +for Slurm's ``gres.conf`` file, use the script provided by this overlay on on the node: + +.. code-block:: shell + + /usr/bin/sh /usr/local/sbin/mig2gres + host ---- diff --git a/warewulf.spec.in b/warewulf.spec.in index b825bc5e..855e1821 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -224,6 +224,7 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %{_overlaydir}/mkswap/rootfs/* %{_overlaydir}/systemd.mount/rootfs/* %{_overlaydir}/systemd.swap/rootfs/* +%{_overlaydir}/mig/rootfs/* %{_bindir}/wwctl %{_prefix}/lib/firewalld/services/warewulf.xml