From cc20a65848014d11323a4d0cd71fff3b1f61d2b1 Mon Sep 17 00:00:00 2001 From: Josh Burks Date: Fri, 30 Jan 2026 14:48:05 -0700 Subject: [PATCH 01/10] Add MIG overlay support with configuration and service files --- overlays/mig/internal/mig_test.go | 112 ++++++++++++++++++ overlays/mig/internal/nodes.conf | 8 ++ .../nvidia-mig.service | 1 + .../etc/systemd/system/nvidia-mig.service.ww | 13 ++ .../mig/rootfs/usr/local/sbin/_config_mig | 37 ++++++ .../mig/rootfs/usr/local/sbin/mig2gres.sh | 30 +++++ 6 files changed, 201 insertions(+) create mode 100644 overlays/mig/internal/mig_test.go create mode 100644 overlays/mig/internal/nodes.conf create mode 120000 overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service create mode 100644 overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww create mode 100644 overlays/mig/rootfs/usr/local/sbin/_config_mig create mode 100644 overlays/mig/rootfs/usr/local/sbin/mig2gres.sh diff --git a/overlays/mig/internal/mig_test.go b/overlays/mig/internal/mig_test.go new file mode 100644 index 00000000..1af93a97 --- /dev/null +++ b/overlays/mig/internal/mig_test.go @@ -0,0 +1,112 @@ +package mig + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/warewulf/warewulf/internal/app/wwctl/overlay/show" + "github.com/warewulf/warewulf/internal/pkg/testenv" + "github.com/warewulf/warewulf/internal/pkg/wwlog" +) + +func Test_migOverlay(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll() + env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf") + env.ImportFile("var/lib/warewulf/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww", "../rootfs/etc/systemd/system/nvidia-mig.service.ww") + + tests := []struct { + name string + args []string + log string + }{ + { + name: "mig:nvidia-mig.service (default)", + args: []string{"--render", "node1", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, + log: migServiceDefault, + }, + { + name: "mig:nvidia-mig.service (per-gpu)", + args: []string{"--render", "node2", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, + log: migServicePerGpu, + }, + { + name: "mig:nvidia-mig.service (custom)", + args: []string{"--render", "node3", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, + log: migServiceCustom, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := show.GetCommand() + cmd.SetArgs(tt.args) + stdout := bytes.NewBufferString("") + stderr := bytes.NewBufferString("") + logbuf := bytes.NewBufferString("") + cmd.SetOut(stdout) + cmd.SetErr(stderr) + wwlog.SetLogWriter(logbuf) + err := cmd.Execute() + assert.NoError(t, err) + assert.Empty(t, stdout.String()) + assert.Empty(t, stderr.String()) + assert.Equal(t, tt.log, logbuf.String()) + }) + } +} + +const migServiceDefault string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/nvidia-mig.service +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/_config_mig "2,2" +ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' + +[Install] +WantedBy=multi-user.target +` + +const migServicePerGpu string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/nvidia-mig.service +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/_config_mig "14,14,14,15 7,7,7,8" +ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' + +[Install] +WantedBy=multi-user.target +` + +const migServiceCustom string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/nvidia-mig.service +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/_config_mig "1,1,1,1" +ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' + +[Install] +WantedBy=multi-user.target +` diff --git a/overlays/mig/internal/nodes.conf b/overlays/mig/internal/nodes.conf new file mode 100644 index 00000000..493c2d31 --- /dev/null +++ b/overlays/mig/internal/nodes.conf @@ -0,0 +1,8 @@ +nodes: + node1: {} + node2: + tags: + gpuMigType: "14,14,14,15 7,7,7,8" + node3: + tags: + gpuMigType: "1,1,1,1" \ No newline at end of file diff --git a/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service b/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service new file mode 120000 index 00000000..fc15c38c --- /dev/null +++ b/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service @@ -0,0 +1 @@ +../nvidia-mig.service \ No newline at end of file diff --git a/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww b/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww new file mode 100644 index 00000000..c632f232 --- /dev/null +++ b/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww @@ -0,0 +1,13 @@ +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/_config_mig "{{ .Tags.gpuMigType | default "2,2" }}" +ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/overlays/mig/rootfs/usr/local/sbin/_config_mig b/overlays/mig/rootfs/usr/local/sbin/_config_mig new file mode 100644 index 00000000..b0d48526 --- /dev/null +++ b/overlays/mig/rootfs/usr/local/sbin/_config_mig @@ -0,0 +1,37 @@ +#!/usr/bin/bash +set -euo pipefail + +# This script configures NVIDIA MIG (Multi-Instance GPU) partitions based on the provided +# gpuMigType string argument. It supports both a single configuration for all GPUs or +# individual configurations per GPU. +# Example inputs: +# "14,14,14,15" # single config for all GPUs +# "14,14,14,15 7,7,7,8" # per-GPU configs for GPU0 and GPU1 + +# See https://docs.nvidia.com/datacenter/tesla/mig-user-guide/supported-mig-profiles.html for details on supported MIG profiles + +# Get the gpuMigType string from the first argument +MIG_STRING="${1}" +if [[ -z "$MIG_STRING" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +# Check if nvidia-smi command exists +if ! command -v nvidia-smi &> /dev/null; then + echo "Error: nvidia-smi command not found" >&2 + exit 1 +fi + +# enable MIG mode +nvidia-smi -mig 1 + +# per-GPU space-separated list (one token per GPU), otherwise single-string common case +if [[ "$MIG_STRING" == *" "* ]]; then + read -a PER_GPU_MIG_CONFIGS <<< "$MIG_STRING" + for gpu_index in "${!PER_GPU_MIG_CONFIGS[@]}"; do + nvidia-smi mig -cgi "${PER_GPU_MIG_CONFIGS[$gpu_index]}" -i "$gpu_index" -C + done +else + nvidia-smi mig -cgi "$MIG_STRING" -C +fi \ No newline at end of file diff --git a/overlays/mig/rootfs/usr/local/sbin/mig2gres.sh b/overlays/mig/rootfs/usr/local/sbin/mig2gres.sh new file mode 100644 index 00000000..a877b152 --- /dev/null +++ b/overlays/mig/rootfs/usr/local/sbin/mig2gres.sh @@ -0,0 +1,30 @@ +#!/usr/bin/bash + +set -euo pipefail + +# Generates NVIDIA MIG device entries for Slurm's gres.conf. +# Queries MIG GPU instances and maps them to /dev/nvidia-caps/ device files. +# +# Process: nvidia-smi -> extract GPU/GI/profile -> lookup minor in +# /proc/driver/nvidia-caps/mig-minors -> map to device file -> output gres format + + +if ! command -v nvidia-smi &> /dev/null; then + echo "Error: nvidia-smi command not found" >&2 + exit 1 +fi + +hostname=$(hostname -s) + +nvidia-smi mig -lgi | grep -E '^\|' | grep -v '===' | grep -v 'GPU.*Name' | while read line; do + gpu=$(echo "$line" | awk '{print $2}') + name=$(echo "$line" | awk '{print $4}') + gi=$(echo "$line" | awk '{print $6}') # Instance ID is column 6 + + [[ "$gpu" =~ ^[0-9]+$ ]] || continue + + pattern="gpu${gpu}/gi${gi}/access" + minor=$(grep "^${pattern} " /proc/driver/nvidia-caps/mig-minors | awk '{print $2}') + + echo "NodeName=${hostname} Name=gpu Type=${name} File=/dev/nvidia-caps/nvidia-cap${minor}" +done \ No newline at end of file From c78f40ec3d8d6102476e111f082316d7c4a6717a Mon Sep 17 00:00:00 2001 From: root Date: Sun, 1 Feb 2026 17:54:52 -0700 Subject: [PATCH 02/10] Move NVIDIA MIG configuration to pure systemd with templating, and colon seperating GPU indexs instead of spaces Signed-off-by: root --- ...idia-mig.service => ww-nvidia-mig.service} | 0 .../etc/systemd/system/nvidia-mig.service.ww | 13 ------- .../systemd/system/ww-nvidia-mig.service.ww | 32 ++++++++++++++++ .../mig/rootfs/usr/local/sbin/_config_mig | 37 ------------------- .../usr/local/sbin/{mig2gres.sh => mig2gres} | 18 ++++++++- 5 files changed, 49 insertions(+), 51 deletions(-) rename overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/{nvidia-mig.service => ww-nvidia-mig.service} (100%) delete mode 100644 overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww create mode 100644 overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww delete mode 100644 overlays/mig/rootfs/usr/local/sbin/_config_mig rename overlays/mig/rootfs/usr/local/sbin/{mig2gres.sh => mig2gres} (67%) diff --git a/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service b/overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/ww-nvidia-mig.service similarity index 100% rename from overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/nvidia-mig.service rename to overlays/mig/rootfs/etc/systemd/system/multi-user.target.wants/ww-nvidia-mig.service diff --git a/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww b/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww deleted file mode 100644 index c632f232..00000000 --- a/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -DefaultDependencies=no -Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time -Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/local/sbin/_config_mig "{{ .Tags.gpuMigType | default "2,2" }}" -ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww new file mode 100644 index 00000000..6983a65a --- /dev/null +++ b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww @@ -0,0 +1,32 @@ +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +{{- $rawMigProfiles := default "2,2" .Tags.gpuMigProfiles -}} +{{- $migProfiles := splitList ":" $rawMigProfiles -}} +{{- if gt (len $migProfiles) 1 -}} +{{- range $gpuIndex, $migProfile := $migProfiles -}} + {{- if ne $migProfile "" }} +ExecStart=/usr/bin/nvidia-smi mig -cgi {{$migProfile}} -i {{$gpuIndex}} -C + {{- end -}} +{{- end -}} +{{- else }} +ExecStart=/usr/bin/nvidia-smi mig -cgi {{$rawMigProfiles}} -C +{{- end }} +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/overlays/mig/rootfs/usr/local/sbin/_config_mig b/overlays/mig/rootfs/usr/local/sbin/_config_mig deleted file mode 100644 index b0d48526..00000000 --- a/overlays/mig/rootfs/usr/local/sbin/_config_mig +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/bash -set -euo pipefail - -# This script configures NVIDIA MIG (Multi-Instance GPU) partitions based on the provided -# gpuMigType string argument. It supports both a single configuration for all GPUs or -# individual configurations per GPU. -# Example inputs: -# "14,14,14,15" # single config for all GPUs -# "14,14,14,15 7,7,7,8" # per-GPU configs for GPU0 and GPU1 - -# See https://docs.nvidia.com/datacenter/tesla/mig-user-guide/supported-mig-profiles.html for details on supported MIG profiles - -# Get the gpuMigType string from the first argument -MIG_STRING="${1}" -if [[ -z "$MIG_STRING" ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -# Check if nvidia-smi command exists -if ! command -v nvidia-smi &> /dev/null; then - echo "Error: nvidia-smi command not found" >&2 - exit 1 -fi - -# enable MIG mode -nvidia-smi -mig 1 - -# per-GPU space-separated list (one token per GPU), otherwise single-string common case -if [[ "$MIG_STRING" == *" "* ]]; then - read -a PER_GPU_MIG_CONFIGS <<< "$MIG_STRING" - for gpu_index in "${!PER_GPU_MIG_CONFIGS[@]}"; do - nvidia-smi mig -cgi "${PER_GPU_MIG_CONFIGS[$gpu_index]}" -i "$gpu_index" -C - done -else - nvidia-smi mig -cgi "$MIG_STRING" -C -fi \ No newline at end of file diff --git a/overlays/mig/rootfs/usr/local/sbin/mig2gres.sh b/overlays/mig/rootfs/usr/local/sbin/mig2gres similarity index 67% rename from overlays/mig/rootfs/usr/local/sbin/mig2gres.sh rename to overlays/mig/rootfs/usr/local/sbin/mig2gres index a877b152..abd389cd 100644 --- a/overlays/mig/rootfs/usr/local/sbin/mig2gres.sh +++ b/overlays/mig/rootfs/usr/local/sbin/mig2gres @@ -14,6 +14,21 @@ if ! command -v nvidia-smi &> /dev/null; then exit 1 fi +if ! command -v grep &> /dev/null; then + echo "Error: grep command not found" >&2 + exit 1 +fi + +if ! command -v awk &> /dev/null; then + echo "Error: awk command not found" >&2 + exit 1 +fi + +if ! command -v hostname &> /dev/null; then + echo "Error: hostname command not found" >&2 + exit 1 +fi + hostname=$(hostname -s) nvidia-smi mig -lgi | grep -E '^\|' | grep -v '===' | grep -v 'GPU.*Name' | while read line; do @@ -25,6 +40,7 @@ nvidia-smi mig -lgi | grep -E '^\|' | grep -v '===' | grep -v 'GPU.*Name' | whil pattern="gpu${gpu}/gi${gi}/access" minor=$(grep "^${pattern} " /proc/driver/nvidia-caps/mig-minors | awk '{print $2}') + device_file="/dev/nvidia-caps/nvidia-cap${minor}" - echo "NodeName=${hostname} Name=gpu Type=${name} File=/dev/nvidia-caps/nvidia-cap${minor}" + echo "NodeName=${hostname} Name=gpu Type=${name} File=${device_file}" done \ No newline at end of file From 2a806b1c51cdc0f5dd9d9c4d008852ceff339867 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 1 Feb 2026 17:55:26 -0700 Subject: [PATCH 03/10] Add tests for mig overlay Signed-off-by: root --- overlays/mig/internal/mig_test.go | 116 +++++++++++++++++++++++------- overlays/mig/internal/nodes.conf | 7 +- 2 files changed, 96 insertions(+), 27 deletions(-) diff --git a/overlays/mig/internal/mig_test.go b/overlays/mig/internal/mig_test.go index 1af93a97..0827d1c9 100644 --- a/overlays/mig/internal/mig_test.go +++ b/overlays/mig/internal/mig_test.go @@ -14,7 +14,7 @@ func Test_migOverlay(t *testing.T) { env := testenv.New(t) defer env.RemoveAll() env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf") - env.ImportFile("var/lib/warewulf/overlays/mig/rootfs/etc/systemd/system/nvidia-mig.service.ww", "../rootfs/etc/systemd/system/nvidia-mig.service.ww") + env.ImportFile("var/lib/warewulf/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww", "../rootfs/etc/systemd/system/ww-nvidia-mig.service.ww") tests := []struct { name string @@ -23,18 +23,23 @@ func Test_migOverlay(t *testing.T) { }{ { name: "mig:nvidia-mig.service (default)", - args: []string{"--render", "node1", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, + args: []string{"--render", "node1", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, log: migServiceDefault, }, { - name: "mig:nvidia-mig.service (per-gpu)", - args: []string{"--render", "node2", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, - log: migServicePerGpu, + name: "mig:nvidia-mig.service (homogeneous)", + args: []string{"--render", "node2", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHomogeneous, }, { - name: "mig:nvidia-mig.service (custom)", - args: []string{"--render", "node3", "mig", "etc/systemd/system/nvidia-mig.service.ww"}, - log: migServiceCustom, + name: "mig:nvidia-mig.service (heterogeneous)", + args: []string{"--render", "node3", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHeterogeneous, + }, + { + name: "mig:nvidia-mig.service (heterogeneous short names)", + args: []string{"--render", "node4", "mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migServiceHeterogeneousShortNames, }, } @@ -59,54 +64,115 @@ func Test_migOverlay(t *testing.T) { const migServiceDefault string = `backupFile: true writeFile: true -Filename: etc/systemd/system/nvidia-mig.service +Filename: etc/systemd/system/ww-nvidia-mig.service [Unit] DefaultDependencies=no -Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time -Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/local/sbin/_config_mig "2,2" -ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 2,2 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 [Install] WantedBy=multi-user.target ` -const migServicePerGpu string = `backupFile: true +const migServiceHomogeneous string = `backupFile: true writeFile: true -Filename: etc/systemd/system/nvidia-mig.service +Filename: etc/systemd/system/ww-nvidia-mig.service [Unit] DefaultDependencies=no -Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time -Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/local/sbin/_config_mig "14,14,14,15 7,7,7,8" -ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 [Install] WantedBy=multi-user.target ` -const migServiceCustom string = `backupFile: true + +const migServiceHeterogeneous string = `backupFile: true writeFile: true -Filename: etc/systemd/system/nvidia-mig.service +Filename: etc/systemd/system/ww-nvidia-mig.service [Unit] DefaultDependencies=no -Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions at boot time -Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/usr/local/sbin/_config_mig "1,1,1,1" -ExecStop=sh -c 'nvidia-smi mig -dci ; nvidia-smi mig -dgi ; nvidia-smi -mig 0' +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 14,14,14,15 -i 0 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 9,9 -i 1 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 [Install] WantedBy=multi-user.target ` + +const migServiceHeterogeneousShortNames string = `backupFile: true +writeFile: true +Filename: etc/systemd/system/ww-nvidia-mig.service +[Unit] +DefaultDependencies=no +Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf +Before=nvidia-persistenced.service nvidia-dcgm.service dcgm_exporter.service nvsm.service nvsm-core.service +ConditionPathExists=/usr/bin/nvidia-smi + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStartPre=-/usr/bin/nvidia-smi -pm 1 +ExecStartPre=/usr/bin/nvidia-smi -mig 1 +ExecStart=/usr/bin/nvidia-smi mig -cgi 1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb -i 0 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 3g.90gb,3g.90gb -i 1 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 2 -C +ExecStart=/usr/bin/nvidia-smi mig -cgi 0 -i 3 -C +ExecStartPost=-/usr/bin/nvidia-smi -L +ExecStartPost=-/usr/bin/sh /usr/local/sbin/mig2gres +ExecStop=-/usr/bin/nvidia-smi mig -dci +ExecStop=-/usr/bin/nvidia-smi mig -dgi +ExecStop=/usr/bin/nvidia-smi -mig 0 +TimeoutStartSec=300 +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target +` \ No newline at end of file diff --git a/overlays/mig/internal/nodes.conf b/overlays/mig/internal/nodes.conf index 493c2d31..88564347 100644 --- a/overlays/mig/internal/nodes.conf +++ b/overlays/mig/internal/nodes.conf @@ -2,7 +2,10 @@ nodes: node1: {} node2: tags: - gpuMigType: "14,14,14,15 7,7,7,8" + gpuMigProfiles: "14,14,14,15" node3: tags: - gpuMigType: "1,1,1,1" \ No newline at end of file + gpuMigProfiles: "14,14,14,15:9,9:0:0" + node4: + tags: + gpuMigProfiles: "1g.23gb+me,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb,1g.23gb:3g.90gb,3g.90gb:0:0" \ No newline at end of file From 2612864a1c58598598194921e651c3578a7054ae Mon Sep 17 00:00:00 2001 From: root Date: Sun, 1 Feb 2026 17:59:14 -0700 Subject: [PATCH 04/10] Add documentation for MIG overlay configuration and usage Signed-off-by: root --- userdocs/overlays/overlays.rst | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/userdocs/overlays/overlays.rst b/userdocs/overlays/overlays.rst index 6e070fec..23bca8e7 100644 --- a/userdocs/overlays/overlays.rst +++ b/userdocs/overlays/overlays.rst @@ -527,6 +527,70 @@ Two overlays, **systemd.mount** and **systemd.swap**, configure mounted and swap storage based on the configuration of native file system fields. They are often paired with the ``mkfs`` and ``mkswap`` overlays. +mig +---- + +The **mig** overlay configures NVIDIA MIG (Multi-Instance GPU) settings on nodes +with supported NVIDIA GPUs. The **mig** overlay requires the NVIDIA driver and +the ``nvidia-smi`` tool to be installed and functional on the node. + +MIG configuration is based on the ``gpuMigProfiles`` tag, which specifies a +comma-separated list of MIG profiles IDs or profile short names to configure. + +For example, to configure a node with four MIG instances of profile 14 and one +instance of profile 15 on all GPUs: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15" + +The MIG instances can also be defined by their profile short names: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb" + +If different GPUs on a node require different MIG configurations, the +``gpuMigProfiles`` tag can be specified as a colon-separated list of comma-separated +lists. Commas separate profiles on a single GPU. Colons separate GPUs by index order. +A value of 0 indicates no MIG partitioning and uses the full GPU. + +For example, to configure a node that has four GPUs, where: + +- GPU 0 has three instances of profile 14 and one instance of profile 15 +- GPU 1 has two instances of profile 9 +- GPU 2 and GPU 3 have one instance of profile 0 (no partitioning): + + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=14,14,14,15:9,9:0:0" + +This can also be done using profile short names: + +.. code-block:: shell + + wwctl node set n1 --tagadd="gpuMigProfiles=2g.20gb,2g.20gb,2g.20gb,1g.20gb:3g.40gb,3g.40gb:0:0" + +The exact MIG configuration available depends on the specific NVIDIA GPU model +present in the node. To see the available MIG profiles for a given GPU model, run: + +.. code-block:: shell + + nvidia-smi mig -lgip + +Refer to `NVIDIA’s documentation `_ for more information on MIG profiles. + +Once configured, the NVIDIA MIG instances can be mapped to Slurm GRES entries +backed by ``/dev/nvidia-caps/capXX`` device files. + +To generate the mapping of MIG instances to ``/dev/nvidia-caps/capXX`` device files +for Slurm's ``gres.conf`` file, use the script provided by this overlay on on the node: + +.. code-block:: shell + + /usr/bin/sh /usr/local/sbin/mig2gres + host ---- From b7adf23703523a9d4c1ac5dc80fa50284d9610b0 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 1 Feb 2026 17:59:36 -0700 Subject: [PATCH 05/10] Add entry for new `mig` overlay in changelog Signed-off-by: root --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f339c3..41d9f955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - New --partwipe flag for profile and node set - Updated arguments for `ValidString` to match `regexp.MatchString` +- New `mig` overlay to configure NVIDIA MIG devices. #2102 ## v4.6.5, 2026-01-12 From c49f28cfddc2ec70c3f4202a26c47d6d4b2442e9 Mon Sep 17 00:00:00 2001 From: Josh Burks Date: Tue, 3 Feb 2026 12:29:01 -0700 Subject: [PATCH 06/10] run make fmt --- overlays/mig/internal/mig_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/overlays/mig/internal/mig_test.go b/overlays/mig/internal/mig_test.go index 0827d1c9..da3377c6 100644 --- a/overlays/mig/internal/mig_test.go +++ b/overlays/mig/internal/mig_test.go @@ -116,7 +116,6 @@ TimeoutStopSec=300 WantedBy=multi-user.target ` - const migServiceHeterogeneous string = `backupFile: true writeFile: true Filename: etc/systemd/system/ww-nvidia-mig.service @@ -175,4 +174,4 @@ TimeoutStopSec=300 [Install] WantedBy=multi-user.target -` \ No newline at end of file +` From d5cbf7972cd5dc27821382bec36342e56188bf1d Mon Sep 17 00:00:00 2001 From: Josh Burks Date: Tue, 3 Feb 2026 16:13:59 -0700 Subject: [PATCH 07/10] add wwdoc string and tests for mig overlay --- overlays/mig/internal/mig_test.go | 49 +++++++++++++++++++ .../systemd/system/ww-nvidia-mig.service.ww | 5 ++ 2 files changed, 54 insertions(+) diff --git a/overlays/mig/internal/mig_test.go b/overlays/mig/internal/mig_test.go index da3377c6..1ba7a0ac 100644 --- a/overlays/mig/internal/mig_test.go +++ b/overlays/mig/internal/mig_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/warewulf/warewulf/internal/app/wwctl/overlay/info" "github.com/warewulf/warewulf/internal/app/wwctl/overlay/show" "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/wwlog" @@ -42,6 +43,19 @@ func Test_migOverlay(t *testing.T) { log: migServiceHeterogeneousShortNames, }, } + infoTests := []struct { + name string + args []string + log string + stdout string + }{ + { + name: "mig:nvidia-mig.service (info)", + args: []string{"mig", "etc/systemd/system/ww-nvidia-mig.service.ww"}, + log: migInfoOutput, + stdout: migInfoStdout, + }, + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -60,11 +74,31 @@ func Test_migOverlay(t *testing.T) { assert.Equal(t, tt.log, logbuf.String()) }) } + + for _, tt := range infoTests { + t.Run(tt.name, func(t *testing.T) { + cmd := info.GetCommand() + cmd.SetArgs(tt.args) + stdout := bytes.NewBufferString("") + stderr := bytes.NewBufferString("") + logbuf := bytes.NewBufferString("") + cmd.SetOut(stdout) + cmd.SetErr(stderr) + wwlog.SetLogWriter(logbuf) + err := cmd.Execute() + assert.NoError(t, err) + assert.Equal(t, tt.log, logbuf.String()) + assert.Equal(t, tt.stdout, stdout.String()) + assert.Empty(t, stderr.String()) + }) + } } const migServiceDefault string = `backupFile: true writeFile: true Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + [Unit] DefaultDependencies=no Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf @@ -92,6 +126,8 @@ WantedBy=multi-user.target const migServiceHomogeneous string = `backupFile: true writeFile: true Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + [Unit] DefaultDependencies=no Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf @@ -119,6 +155,8 @@ WantedBy=multi-user.target const migServiceHeterogeneous string = `backupFile: true writeFile: true Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + [Unit] DefaultDependencies=no Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf @@ -149,6 +187,8 @@ WantedBy=multi-user.target const migServiceHeterogeneousShortNames string = `backupFile: true writeFile: true Filename: etc/systemd/system/ww-nvidia-mig.service +# This file is autogenerated by warewulf + [Unit] DefaultDependencies=no Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf @@ -175,3 +215,12 @@ TimeoutStopSec=300 [Install] WantedBy=multi-user.target ` + +const migInfoOutput string = `Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node. +` + +const migInfoStdout string = ` +VARIABLE OPTION TYPE HELP +-------- ------ ---- ---- +.Tags.gpuMigProfiles string List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index +` diff --git a/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww index 6983a65a..34d89e30 100644 --- a/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww +++ b/overlays/mig/rootfs/etc/systemd/system/ww-nvidia-mig.service.ww @@ -1,3 +1,8 @@ +# This file is autogenerated by warewulf + +{{- /* wwdoc: Configures NVIDIA MIG (Multi-Instance GPU) settings on nodes with supported NVIDIA GPUs. Requires the NVIDIA driver and the nvidia-smi tool to be installed and functional on the node. */}} +{{- /* .Tags.gpuMigProfiles: List of MIG profile IDs or short names, with commas separating instances on a GPU and colons separating GPUs by index */}} + [Unit] DefaultDependencies=no Description=Configure NVIDIA MIG (Multi-Instance GPU) partitions via Warewulf From 89b25e27505b4f3dbde3b9e59daccbbc3e3608c3 Mon Sep 17 00:00:00 2001 From: Josh Burks Date: Tue, 3 Feb 2026 16:21:24 -0700 Subject: [PATCH 08/10] Add mig overlay directory to package specification --- warewulf.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/warewulf.spec.in b/warewulf.spec.in index b825bc5e..855e1821 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -224,6 +224,7 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %{_overlaydir}/mkswap/rootfs/* %{_overlaydir}/systemd.mount/rootfs/* %{_overlaydir}/systemd.swap/rootfs/* +%{_overlaydir}/mig/rootfs/* %{_bindir}/wwctl %{_prefix}/lib/firewalld/services/warewulf.xml From 6efbe3e0afe81693254a5c2ab492ac1b1f5ff081 Mon Sep 17 00:00:00 2001 From: Sujeev-Uthayakumar Date: Mon, 2 Feb 2026 08:59:39 -0500 Subject: [PATCH 09/10] update contributors and changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d9f955..a82ad32a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - New --partwipe flag for profile and node set - Updated arguments for `ValidString` to match `regexp.MatchString` - New `mig` overlay to configure NVIDIA MIG devices. #2102 +- Updated arguments for `ValidString` to match `regexp.MatchString` ## v4.6.5, 2026-01-12 From 3c58c556aec3fc6e39d46dfca02b26b50937f5e5 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 1 Feb 2026 17:59:36 -0700 Subject: [PATCH 10/10] Add entry for new `mig` overlay in changelog Signed-off-by: root --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a82ad32a..f6030044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Updated arguments for `ValidString` to match `regexp.MatchString` - New `mig` overlay to configure NVIDIA MIG devices. #2102 - Updated arguments for `ValidString` to match `regexp.MatchString` +- New `mig` overlay to configure NVIDIA MIG devices. #2102 ## v4.6.5, 2026-01-12