From a322061fe9345eb43766b135bb68e02c1a957b84 Mon Sep 17 00:00:00 2001 From: xu yang Date: Tue, 16 Apr 2024 10:55:23 -0600 Subject: [PATCH 1/7] Add initramfs stage to warewulfd The initramfs stage supports serving an initramfs image from within the container image, typically generated by dracut. Signed-off-by: xu yang Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + internal/pkg/container/config.go | 27 ++++++++ internal/pkg/container/config_test.go | 88 ++++++++++++++++++++++++ internal/pkg/warewulfd/parser.go | 2 + internal/pkg/warewulfd/provision.go | 26 +++++-- internal/pkg/warewulfd/provision_test.go | 6 ++ 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 internal/pkg/container/config_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bea5495a..1e4effd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Add examples for building overlays in parallel to documentation +- Add `stage=initramfs` to warewulfd provision to serve initramfs from container image. #1115 ### Changed diff --git a/internal/pkg/container/config.go b/internal/pkg/container/config.go index 350cef24..902827ea 100644 --- a/internal/pkg/container/config.go +++ b/internal/pkg/container/config.go @@ -1,11 +1,25 @@ package container import ( + "fmt" "path" + "github.com/warewulf/warewulf/internal/pkg/util" + "github.com/warewulf/warewulf/internal/pkg/wwlog" + warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" ) +var ( + initramfsSearchPaths = []string{ + // This is a printf format where the %s will be the kernel version + "boot/initramfs-%s", + "boot/initramfs-%s.img", + "boot/initrd-%s", + "boot/initrd-%s.img", + } +) + func SourceParentDir() string { conf := warewulfconf.Get() return conf.Paths.WWChrootdir @@ -27,3 +41,16 @@ func ImageParentDir() string { func ImageFile(name string) string { return path.Join(ImageParentDir(), name+".img") } + +// InitramfsBootPath returns the dracut built initramfs path, as dracut built initramfs inside container +// the function returns host path of the built file +func InitramfsBootPath(image, kver string) (string, error) { + for _, searchPath := range initramfsSearchPaths { + initramfs_path := path.Join(RootFsDir(image), fmt.Sprintf(searchPath, kver)) + wwlog.Debug("Looking for initramfs at: %s", initramfs_path) + if util.IsFile(initramfs_path) { + return initramfs_path, nil + } + } + return "", fmt.Errorf("Failed to find a target kernel version initramfs") +} diff --git a/internal/pkg/container/config_test.go b/internal/pkg/container/config_test.go new file mode 100644 index 00000000..904bc6b7 --- /dev/null +++ b/internal/pkg/container/config_test.go @@ -0,0 +1,88 @@ +package container + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" +) + +func TestInitramfsBootPath(t *testing.T) { + conf := warewulfconf.Get() + temp, err := os.MkdirTemp(os.TempDir(), "ww-conf-*") + assert.NoError(t, err) + defer os.RemoveAll(temp) + conf.Paths.WWChrootdir = temp + + assert.NoError(t, os.MkdirAll(filepath.Join(RootFsDir("image"), "boot"), 0700)) + + tests := []struct { + name string + initramfs []string + ver string + err error + retName string + }{ + { + name: "ok case 1", + initramfs: []string{"initramfs-1.1.1.aarch64.img"}, + ver: "1.1.1.aarch64", + err: nil, + }, + { + name: "ok case 2", + initramfs: []string{"initrd-1.1.1.aarch64"}, + ver: "1.1.1.aarch64", + err: nil, + }, + { + name: "ok case 3", + initramfs: []string{"initramfs-1.1.1.aarch64"}, + ver: "1.1.1.aarch64", + err: nil, + }, + { + name: "ok case 4", + initramfs: []string{"initrd-1.1.1.aarch64.img"}, + ver: "1.1.1.aarch64", + err: nil, + }, + { + name: "error case, wrong init name", + initramfs: []string{"initrr-1.1.1.aarch64.img"}, + ver: "1.1.1.aarch64", + err: fmt.Errorf("Failed to find a target kernel version initramfs"), + }, + { + name: "error case, wrong ver", + initramfs: []string{"initrr-1.1.1.aarch64.img"}, + ver: "1.1.2.aarch64", + err: fmt.Errorf("Failed to find a target kernel version initramfs"), + }, + } + + for _, tt := range tests { + t.Logf("running test: %s", tt.name) + for _, init := range tt.initramfs { + assert.NoError(t, os.WriteFile(filepath.Join(RootFsDir("image"), "boot", init), []byte(""), 0600)) + } + initPath, err := InitramfsBootPath("image", tt.ver) + assert.Equal(t, tt.err, err) + if err == nil { + assert.NotEmpty(t, initPath) + } else { + assert.Empty(t, initPath) + } + + if tt.retName != "" { + assert.Equal(t, filepath.Base(initPath), tt.retName) + } + // remove the file + for _, init := range tt.initramfs { + assert.NoError(t, os.Remove(filepath.Join(RootFsDir("image"), "boot", init))) + } + } +} diff --git a/internal/pkg/warewulfd/parser.go b/internal/pkg/warewulfd/parser.go index b91b9cdc..e542982f 100644 --- a/internal/pkg/warewulfd/parser.go +++ b/internal/pkg/warewulfd/parser.go @@ -73,6 +73,8 @@ func parseReq(req *http.Request) (parserInfo, error) { ret.stage = "runtime" } else if stage == "efiboot" { ret.stage = "efiboot" + } else if stage == "initramfs" { + ret.stage = "initramfs" } } diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 0d544d69..8861cd99 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -56,12 +56,13 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } status_stages := map[string]string{ - "efiboot": "EFI", - "ipxe": "IPXE", - "kernel": "KERNEL", - "kmods": "KMODS_OVERLAY", - "system": "SYSTEM_OVERLAY", - "runtime": "RUNTIME_OVERLAY"} + "efiboot": "EFI", + "ipxe": "IPXE", + "kernel": "KERNEL", + "kmods": "KMODS_OVERLAY", + "system": "SYSTEM_OVERLAY", + "runtime": "RUNTIME_OVERLAY", + "initramfs": "INITRAMFS"} status_stage := status_stages[rinfo.stage] var stage_file string @@ -213,6 +214,19 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } else { wwlog.Warn("No conainer set for node %s", node.Id.Get()) } + } else if rinfo.stage == "initramfs" { + if node.ContainerName.Defined() { + _, kver, err := kernel.FindKernel(container.RootFsDir(node.ContainerName.Get())) + if err != nil { + wwlog.Error("No kernel found for initramfs for container %s: %s", node.ContainerName.Get(), err) + } + stage_file, err = container.InitramfsBootPath(node.ContainerName.Get(), kver) + if err != nil { + wwlog.Error("No initramfs found for container %s: %s", node.ContainerName.Get(), err) + } + } else { + wwlog.Warn("No container set for node %s", node.Id.Get()) + } } wwlog.Serv("stage_file '%s'", stage_file) diff --git a/internal/pkg/warewulfd/provision_test.go b/internal/pkg/warewulfd/provision_test.go index 1b1446a9..27ad6d33 100644 --- a/internal/pkg/warewulfd/provision_test.go +++ b/internal/pkg/warewulfd/provision_test.go @@ -30,6 +30,7 @@ var provisionSendTests = []struct { {"find shim", "/efiboot/shim.efi", "", 404, "10.10.10.11:9873"}, {"find grub", "/efiboot/grub.efi", "", 200, "10.10.10.10:9873"}, {"find grub", "/efiboot/grub.efi", "", 404, "10.10.10.11:9873"}, + {"find initramfs", "/provision/00:00:00:ff:ff:ff?stage=initramfs", "", 200, "10.10.10.10:9873"}, } func Test_ProvisionSend(t *testing.T) { @@ -84,6 +85,11 @@ nodes: _, err := os.Create(path.Join(containerDir, "suse/rootfs/usr/share/efi/x86_64/", "grub.efi")) assert.NoError(t, err) } + assert.NoError(t, os.MkdirAll(path.Join(containerDir, "suse/rootfs/boot"), 0700)) + { + _, err := os.Create(path.Join(containerDir, "suse/rootfs/boot", "initramfs-.img")) + assert.NoError(t, err) + } dbErr := LoadNodeDB() assert.NoError(t, dbErr) From 18cebb86525de9854b4c6499e47dca9736d794dc Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Tue, 16 Apr 2024 10:57:58 -0600 Subject: [PATCH 2/7] New warewulf-dracut subpackage warewulf-dracut includes a dracut module to be installed in a container image to support building an initramfs that can boot from Warewulf. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + Makefile | 2 ++ Variables.mk | 3 ++- dracut/modules.d/90wwinit/load-wwinit.sh | 13 ++++++++++ dracut/modules.d/90wwinit/module-setup.sh | 20 ++++++++++++++++ dracut/modules.d/90wwinit/parse-wwinit.sh | 28 ++++++++++++++++++++++ warewulf.spec.in | 29 ++++++++++++++++++++++- 7 files changed, 94 insertions(+), 2 deletions(-) create mode 100755 dracut/modules.d/90wwinit/load-wwinit.sh create mode 100755 dracut/modules.d/90wwinit/module-setup.sh create mode 100755 dracut/modules.d/90wwinit/parse-wwinit.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4effd5..95c7369f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add examples for building overlays in parallel to documentation - Add `stage=initramfs` to warewulfd provision to serve initramfs from container image. #1115 +- Add `warewulf-dracut` package to support building Warewulf-compatible initramfs images with dracut. #1115 ### Changed diff --git a/Makefile b/Makefile index 3d8cf23a..8a8eaed7 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,8 @@ install: build docs install -m 0644 etc/bash_completion.d/wwctl $(DESTDIR)$(BASHCOMPDIR)/wwctl for f in docs/man/man1/*.1.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man1/; done for f in docs/man/man5/*.5.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man5/; done + install -pd -m 0755 $(DESTDIR)$(DRACUTMODDIR)/90wwinit + install -m 0644 dracut/modules.d/90wwinit/*.sh $(DESTDIR)$(DRACUTMODDIR)/90wwinit .PHONY: installapi installapi: diff --git a/Variables.mk b/Variables.mk index b6d17784..b23f6a11 100644 --- a/Variables.mk +++ b/Variables.mk @@ -44,10 +44,11 @@ else endif # OS-Specific Service Locations -VARLIST += TFTPDIR FIREWALLDDIR SYSTEMDDIR BASHCOMPDIR +VARLIST += TFTPDIR FIREWALLDDIR SYSTEMDDIR BASHCOMPDIR DRACUTMODDIR SYSTEMDDIR ?= /usr/lib/systemd/system BASHCOMPDIR ?= /etc/bash_completion.d FIREWALLDDIR ?= /usr/lib/firewalld/services +DRACUTMODDIR ?= /usr/lib/dracut/modules.d ifeq ($(OS),suse) TFTPDIR ?= /srv/tftpboot endif diff --git a/dracut/modules.d/90wwinit/load-wwinit.sh b/dracut/modules.d/90wwinit/load-wwinit.sh new file mode 100755 index 00000000..3086207d --- /dev/null +++ b/dracut/modules.d/90wwinit/load-wwinit.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +info "Mounting tmpfs at $NEWROOT" +mount -t tmpfs ${wwinit_tmpfs_size_option} tmpfs "$NEWROOT" + +for archive in "${wwinit_container}" "${wwinit_kmods}" "${wwinit_system}" "${wwinit_runtime}" +do + if [ -n "${archive}" ] + then + info "Loading ${archive}" + (curl --silent -L "${archive}" | gzip -d | cpio -im --directory="${NEWROOT}") || die "Unable to load ${archive}" + fi +done diff --git a/dracut/modules.d/90wwinit/module-setup.sh b/dracut/modules.d/90wwinit/module-setup.sh new file mode 100755 index 00000000..454562ad --- /dev/null +++ b/dracut/modules.d/90wwinit/module-setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +check() { + # Don't include in hostonly mode + [[ $hostonly ]] && return 1 + + # Don't include by default + return 255 +} + +depends() { + echo network + return 0 +} + +install() { + inst_multiple cpio curl + inst_hook cmdline 30 "$moddir/parse-wwinit.sh" + inst_hook pre-mount 30 "$moddir/load-wwinit.sh" +} diff --git a/dracut/modules.d/90wwinit/parse-wwinit.sh b/dracut/modules.d/90wwinit/parse-wwinit.sh new file mode 100755 index 00000000..45875024 --- /dev/null +++ b/dracut/modules.d/90wwinit/parse-wwinit.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# root=wwinit + +[ -z "$root" ] && root=$(getarg root=) + +if [ "${root}" = "wwinit" ] +then + info "root=${root}" + export wwinit_container=$(getarg wwinit.container=); info "wwinit.container=${wwinit_container}" + export wwinit_system=$(getarg wwinit.system=); info "wwinit.system=${wwinit_system}" + export wwinit_runtime=$(getarg wwinit.runtime=); info "wwinit.runtime=${wwinit_runtime}" + export wwinit_kmods=$(getarg wwinit.kmods=); info "wwinit.kmods=${wwinit_kmods}" + + wwinit_tmpfs_size=$(getarg wwinit.tmpfs.size=) + if [ -n "$wwinit_tmpfs_size" ] + then + info "wwinit.tmpfs.size=${wwinit_tmpfs_size}" + export wwinit_tmpfs_size_option="-o size=${wwinit_tmpfs_size}" + fi + + if [ -n "${wwinit_container}" ] + then + info "Found root=${root} and a Warewulf container image. Will boot from Warewulf." + rootok=1 + else + die "Found root=${root} but no container image. Cannot boot from Warewulf." + fi +fi diff --git a/warewulf.spec.in b/warewulf.spec.in index 71e6bbf2..63a1fddb 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -82,6 +82,23 @@ BuildRequires: libassuan-devel gpgme-devel Warewulf is a stateless and diskless container operating system provisioning system for large clusters of bare metal and/or virtual systems. +%package dracut +Summary: dracut module for loading a Warewulf container image +BuildArch: noarch + +Requires: dracut +%if 0%{?suse_version} +%else +Requires: dracut-network +%endif + +%description dracut +Warewulf is a stateless and diskless container operating system provisioning +system for large clusters of bare metal and/or virtual systems. + +This subpackage contains a dracut module that can be used to generate +an initramfs that can fetch and boot a Warewulf container image from a +Warewulf server. %prep %setup -q -n %{name}-%{version} -b0 %if %{?with_offline:-a2} @@ -105,7 +122,8 @@ make defaults \ BASHCOMPDIR=/etc/bash_completion.d/ \ FIREWALLDDIR=/usr/lib/firewalld/services \ WWCLIENTDIR=/warewulf \ - IPXESOURCE=/usr/share/ipxe + IPXESOURCE=/usr/share/ipxe \ + DRACUTMODDIR=/usr/lib/dracut/modules.d make %if %{api} make api @@ -185,7 +203,16 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %endif +%files dracut +%defattr(-, root, root) +%dir %{_prefix}/lib/dracut/modules.d/90wwinit +%{_prefix}/lib/dracut/modules.d/90wwinit/*.sh + + %changelog +* Thu Apr 18 2024 Jonathon Anderson +- New warewulf-dracut subpackage + * Wed Apr 17 2024 Jonathon Anderson - Don't build the API on EL7 From 53dae60e20fe82ee4fe1052e0bc1ebeb9c0904f2 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Tue, 16 Apr 2024 11:00:54 -0600 Subject: [PATCH 3/7] New, optional dracut.ipxe configuration dracut.ipxe boots an initramfs detected in the container image. This required a few small changes to warewulfd and the wwinit overlay to accommodate the dracut initramfs configuring the network interface before wwinit: - Include .NetDevs in the ipxe configuration template, so that dracut can configure interfaces with the correct names. - Configure NetworkManager to not keep the initramfs configuration, in stead applying the configuration found by wwinit. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 3 ++ etc/ipxe/dracut.ipxe | 46 +++++++++++++++++++ internal/pkg/warewulfd/provision.go | 6 ++- .../conf.d/ww4-discard-configuration.conf | 2 + 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 etc/ipxe/dracut.ipxe create mode 100644 overlays/wwinit/rootfs/etc/NetworkManager/conf.d/ww4-discard-configuration.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c7369f..8efcb914 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,11 +40,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add examples for building overlays in parallel to documentation - Add `stage=initramfs` to warewulfd provision to serve initramfs from container image. #1115 - Add `warewulf-dracut` package to support building Warewulf-compatible initramfs images with dracut. #1115 +- Add iPXE template `dracut.ipxe` to boot a dracut initramfs. #1115 +- Add `.NetDevs` variable to iPXE templates, similar to overlay templates. #1115 ### Changed - Replace reference to docusaurus with Sphinx - `wwctl container import` now only runs syncuser if explicitly requested. #1212 +- wwinit now configures NetworkManager to not retain configurations from dracut. #1115 ### Fixed diff --git a/etc/ipxe/dracut.ipxe b/etc/ipxe/dracut.ipxe new file mode 100644 index 00000000..4336b01e --- /dev/null +++ b/etc/ipxe/dracut.ipxe @@ -0,0 +1,46 @@ +#!ipxe + +echo +echo ================================================================================ +echo Warewulf v4 now booting via dracut: {{.Fqdn}} ({{.Hwaddr}}) +echo +echo Container: {{.ContainerName}} +{{if .KernelOverride }} +echo Kernel: {{.KernelOverride}} +{{else}} +echo Kernel: {{.ContainerName}} (container default) +{{end}} +echo KernelArgs: {{.KernelArgs}} +echo + +set uri http://{{.Ipaddr}}:{{.Port}}/provision/{{.Hwaddr}}?assetkey=${asset}&uuid=${uuid} +echo Warewulf Controller: {{.Ipaddr}} + +echo Downloading Kernel Image: +kernel --name kernel ${uri}&stage=kernel || goto reboot + +{{if ne .KernelOverride ""}} +echo Downloading Kernel Modules: +imgextract --name kmods ${uri}&stage=kmods&compress=gz || initrd --name kmods ${uri}&stage=kmods || goto reboot +set kernel_mods initrd=kmods +{{end}} + +echo Downloading initramfs +initrd --name initramfs ${uri}&stage=initramfs || goto reboot + +set dracut_net rd.neednet=1 {{range $devname, $netdev := .NetDevs}}{{if and $netdev.Hwaddr $netdev.Device}} ifname={{$netdev.Device}}:{{$netdev.Hwaddr}} {{end}}{{end}} +set dracut_wwinit root=wwinit wwinit.container=${uri}&stage=container&compress=gz wwinit.system=${uri}&stage=system&compress=gz wwinit.runtime=${uri}&stage=runtime&compress=gz {{if ne .KernelOverride ""}}wwinit.kmods=${uri}&stage=kmods&compress=gz{{end}} init=/init + +echo Booting initramfs +#echo Network KernelArgs: ${dracut_net} +#echo Dracut wwinit KernelArgs: ${dracut_wwinit} +#sleep 15 +boot kernel initrd=initramfs ${kernel_mods} ${dracut_net} ${dracut_wwinit} wwid={{.Hwaddr}} {{.KernelArgs}} + + +:reboot +echo +echo There was an error, rebooting in 15s... +echo +sleep 15 +reboot diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 8861cd99..9ef089ba 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -15,6 +15,7 @@ import ( warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/container" "github.com/warewulf/warewulf/internal/pkg/kernel" + "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/overlay" "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" @@ -33,6 +34,7 @@ type templateVars struct { Port string KernelArgs string KernelOverride string + NetDevs map[string]*node.NetDevs } func ProvisionSend(w http.ResponseWriter, req *http.Request) { @@ -94,6 +96,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } else if rinfo.stage == "ipxe" { stage_file = path.Join(conf.Paths.Sysconfdir, "warewulf/ipxe/"+node.Ipxe.Get()+".ipxe") + tstruct := overlay.InitStruct(&node) tmpl_data = templateVars{ Id: node.Id.Get(), Cluster: node.ClusterName.Get(), @@ -104,7 +107,8 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { Hwaddr: rinfo.hwaddr, ContainerName: node.ContainerName.Get(), KernelArgs: node.Kernel.Args.Get(), - KernelOverride: node.Kernel.Override.Get()} + KernelOverride: node.Kernel.Override.Get(), + NetDevs: tstruct.NetDevs} } else if rinfo.stage == "kernel" { if node.Kernel.Override.Defined() { stage_file = kernel.KernelImage(node.Kernel.Override.Get()) diff --git a/overlays/wwinit/rootfs/etc/NetworkManager/conf.d/ww4-discard-configuration.conf b/overlays/wwinit/rootfs/etc/NetworkManager/conf.d/ww4-discard-configuration.conf new file mode 100644 index 00000000..8a7b064a --- /dev/null +++ b/overlays/wwinit/rootfs/etc/NetworkManager/conf.d/ww4-discard-configuration.conf @@ -0,0 +1,2 @@ +[device] +keep-configuration=no From cb5146b09815cce8cd0822c8827e0f720c6efcc1 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Tue, 16 Apr 2024 11:04:45 -0600 Subject: [PATCH 4/7] User documentation for dracut initramfs booting Signed-off-by: Jonathon Anderson --- userdocs/contents/boot-management.rst | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/userdocs/contents/boot-management.rst b/userdocs/contents/boot-management.rst index 79cc3c30..d7bcec23 100644 --- a/userdocs/contents/boot-management.rst +++ b/userdocs/contents/boot-management.rst @@ -5,6 +5,9 @@ Boot Management Warewulf uses iPXE to for network boot by default. As a tech preview, support for GRUB is also available, which adds support for secure boot. +Also as a tech preview, Warewulf may also use iPXE to boot a dracut +initramfs as an initial stage before loading the container image. + Booting with iPXE ================= @@ -29,6 +32,64 @@ Booting with iPXE ipxe_cfg->kernel[ltail=cluster0,label="http"]; } +Booting with dracut +------------------- + +Some systems, typically due to limitations in their BIOS or EFI +firmware, are unable to load container image of a certain size +directly with a traditional bootloader, either iPXE or GRUB. As a +workaround for such systems, Warewulf can be configured to load a +dracut initramfs from the container and to use that initramfs to load +the full container image. + +Warewulf provides a dracut module to configure the dracut initramfs to +load the container image. This module is available in the +``warewulf-dracut`` subpackage, which must be installed in the +container image. + +With the ``warewulf-dracut`` package installed, you can build an +initramfs inside the container. + +.. code-block:: shell + + dnf -y install warewulf-dracut + dracut --force --no-hostonly --add wwinit --kver $(ls /lib/modules | head -n1) + +Set the node's iPXE template to ``dracut`` to direct iPXE to fetch the +node's initramfs image and boot with dracut semantics, rather than +booting the node image directly. + +.. note:: + + Warewulf iPXE templates are located at ``/etc/warewulf/ipxe/`` when + Warewulf is installed via official packages. You can learn more + about how dracut booting works by inspecting its iPXE template at + ``/etc/warewulf/ipxe/dracut.ipxe``. + +.. code-block:: shell + + wwctl node set wwnode1 --ipxe dracut + +.. note:: + + The iPXE template may be set at the node or profile level. + +During boot, ``warewulfd`` will detect and dynamically serve an +initramfs from a node's container image in much the same way that it +can serve a kernel from a container image. This image is loaded by +iPXE, and iPXE directs dracut to fetch the node's container image +during boot. + +The wwinit module provisions to tmpfs. By default, tmpfs is permitted +to use up to 50% of physical memory. This size limit may be adjustd +using the kernel argument `wwinit.tmpfs.size`. (This parameter is +passed to the `size` option during tmpfs mount. See ``tmpfs(5)`` for +more details.) + +.. warning:: + + Kernel overrides are not currently supported during dracut initramfs boot. + Booting with GRUB ================= From 48d6b6303049eb9c3c7833fbf69f3f8ac4f144a2 Mon Sep 17 00:00:00 2001 From: xu yang Date: Thu, 18 Apr 2024 16:18:17 -0600 Subject: [PATCH 5/7] Add tests for new iPXE template variables Signed-off-by: Jonathon Anderson --- internal/pkg/warewulfd/provision_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/pkg/warewulfd/provision_test.go b/internal/pkg/warewulfd/provision_test.go index 27ad6d33..4ef4437a 100644 --- a/internal/pkg/warewulfd/provision_test.go +++ b/internal/pkg/warewulfd/provision_test.go @@ -31,6 +31,7 @@ var provisionSendTests = []struct { {"find grub", "/efiboot/grub.efi", "", 200, "10.10.10.10:9873"}, {"find grub", "/efiboot/grub.efi", "", 404, "10.10.10.11:9873"}, {"find initramfs", "/provision/00:00:00:ff:ff:ff?stage=initramfs", "", 200, "10.10.10.10:9873"}, + {"ipxe test with NetDevs and KernelOverrides", "/provision/00:00:00:00:00:ff?stage=ipxe", "1.1.1 ifname=net:00:00:00:00:00:ff ", 200, "10.10.10.12:9873"}, } func Test_ProvisionSend(t *testing.T) { @@ -51,7 +52,15 @@ nodes: network devices: default: hwaddr: 00:00:00:00:ff:ff - container name: none`) + container name: none + n3: + network devices: + default: + hwaddr: 00:00:00:00:00:ff + device: net + ipxe template: test + kernel: + override: 1.1.1`) assert.NoError(t, err) } assert.NoError(t, conf_file.Sync()) @@ -64,7 +73,8 @@ nodes: { _, err := arp_file.WriteString(`IP address HW type Flags HW address Mask Device 10.10.10.10 0x1 0x2 00:00:00:ff:ff:ff * dummy -10.10.10.11 0x1 0x2 00:00:00:00:ff:ff * dummy`) +10.10.10.11 0x1 0x2 00:00:00:00:ff:ff * dummy +10.10.10.12 0x1 0x2 00:00:00:00:00:ff * dummy`) assert.NoError(t, err) } assert.NoError(t, arp_file.Sync()) @@ -75,6 +85,12 @@ nodes: assert.NoError(t, imageDirErr) defer os.RemoveAll(containerDir) conf.Paths.WWChrootdir = containerDir + + sysConfDir, sysConfDirErr := os.MkdirTemp(os.TempDir(), "ww-test-sysconf-*") + assert.NoError(t, sysConfDirErr) + defer os.RemoveAll(sysConfDir) + conf.Paths.Sysconfdir = sysConfDir + assert.NoError(t, os.MkdirAll(path.Join(containerDir, "suse/rootfs/usr/lib64/efi"), 0700)) { _, err := os.Create(path.Join(containerDir, "suse/rootfs/usr/lib64/efi", "shim.efi")) @@ -90,6 +106,10 @@ nodes: _, err := os.Create(path.Join(containerDir, "suse/rootfs/boot", "initramfs-.img")) assert.NoError(t, err) } + assert.NoError(t, os.MkdirAll(path.Join(conf.Paths.Sysconfdir, "warewulf/ipxe"), 0700)) + { + assert.NoError(t, os.WriteFile(path.Join(conf.Paths.Sysconfdir, "warewulf/ipxe", "test.ipxe"), []byte("{{.KernelOverride}}{{range $devname, $netdev := .NetDevs}}{{if and $netdev.Hwaddr $netdev.Device}} ifname={{$netdev.Device}}:{{$netdev.Hwaddr}} {{end}}{{end}}"), 0600)) + } dbErr := LoadNodeDB() assert.NoError(t, dbErr) From 9500487ad39e71505c5e5a61b75ad6f1e0abdda7 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 18 Apr 2024 16:36:01 -0600 Subject: [PATCH 6/7] Improve the detection of SELinux capable rootfs initramfs uses tmpfs regardless of whether root=tmpfs in Warewulf. As such, 90-selinux needs to be updated to detect the actual provenance of /, rather than the configured setting. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + .../wwinit/rootfs/warewulf/init.d/90-selinux | 36 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8efcb914..eaed83f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Replace reference to docusaurus with Sphinx - `wwctl container import` now only runs syncuser if explicitly requested. #1212 - wwinit now configures NetworkManager to not retain configurations from dracut. #1115 +- Improved detection of SELinux capable root fs #1093 ### Fixed diff --git a/overlays/wwinit/rootfs/warewulf/init.d/90-selinux b/overlays/wwinit/rootfs/warewulf/init.d/90-selinux index 73ad892d..2b910b2e 100644 --- a/overlays/wwinit/rootfs/warewulf/init.d/90-selinux +++ b/overlays/wwinit/rootfs/warewulf/init.d/90-selinux @@ -2,11 +2,6 @@ . /warewulf/config -if test -z "$WWROOT"; then - echo "Skipping SELinux configuration: Warewulf Root device not set" - exit -fi - if test -f "/etc/sysconfig/selinux"; then . /etc/sysconfig/selinux else @@ -14,22 +9,25 @@ else exit fi -if test "$WWROOT" == "initramfs"; then - echo "Skipping SELinux configuration: 'Root=initramfs'" - if test "$SELINUX" != "disabled"; then - echo "WARNING: SELinux prep is being skipped, but SELinux is enabled on host! This may" - echo "WARNING: cause the system to not work properly. Try setting 'Root=tmpfs'" - sleep 5 - fi +if test "$SELINUX" == "disabled"; then + echo "Skipping SELinux setup per /etc/sysconfig/selinux" exit fi -if test "$SELINUX" == "disabled"; then - echo "Skipping SELinux setup per /etc/sysconfig/selinux" -elif grep -q "selinux=0" /proc/cmdline; then +if grep -q "selinux=0" /proc/cmdline; then echo "Skipping SELinux setup per kernel command line" -else - echo "Setting up SELinux" - /sbin/load_policy -i - /sbin/restorecon -r / + exit fi + +if [ $(findmnt / --noheadings --output SOURCE) == 'rootfs' ]; then + echo "Skipping SELinux configuration: rootfs does not support SELinux contexts" + echo + echo "WARNING: SELinux prep is being skipped, but SELinux is enabled on host! This may" + echo "WARNING: cause the system to not work properly. Try setting 'Root=tmpfs'" + sleep 5 + exit +fi + +echo "Setting up SELinux" +/sbin/load_policy -i +/sbin/restorecon -e /sys -r / From 05b951d78da284b20035f69381a4bde03e6af9ed Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 6 Jun 2024 03:08:43 -0400 Subject: [PATCH 7/7] Support dracut booting with GRUB Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 4 ++- etc/grub/grub.cfg.ww | 42 ++++++++++++++++++++++++++- internal/pkg/warewulfd/provision.go | 9 ++++-- userdocs/contents/boot-management.rst | 21 ++++++++++++-- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaed83f5..b44547f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add `stage=initramfs` to warewulfd provision to serve initramfs from container image. #1115 - Add `warewulf-dracut` package to support building Warewulf-compatible initramfs images with dracut. #1115 - Add iPXE template `dracut.ipxe` to boot a dracut initramfs. #1115 -- Add `.NetDevs` variable to iPXE templates, similar to overlay templates. #1115 +- Add dracut menuentry to `grub.cfg.ww` to boot a dracut initramfs. #1115 +- Add `.NetDevs` variable to iPXE and GRUB templates, similar to overlay templates. #1115 +- Add `.Tags` variable to iPXE and GRUB templates, similar to overlay templates. #1115 ### Changed diff --git a/etc/grub/grub.cfg.ww b/etc/grub/grub.cfg.ww index dcb806a0..f7ecac23 100644 --- a/etc/grub/grub.cfg.ww +++ b/etc/grub/grub.cfg.ww @@ -6,13 +6,16 @@ echo "Warewulf Controller: {{.Ipaddr}}" echo sleep 1 smbios --type1 --get-string 8 --set assetkey + uri="(http,{{.Ipaddr}}:{{.Port}})/provision/${net_default_mac}?assetkey=${assetkey}" kernel="${uri}&stage=kernel" container="${uri}&stage=container&compress=gz" system="${uri}&stage=system&compress=gz" runtime="${uri}&stage=runtime&compress=gz" -set default=ww4 + +set default={{ or .Tags.GrubMenuEntry "ww4" }} set timeout=5 + menuentry "Network boot node: {{.Id}}" --id ww4 { {{if .KernelOverride }} echo "Kernel: {{.KernelOverride}}" @@ -34,17 +37,54 @@ menuentry "Network boot node: {{.Id}}" --id ww4 { reboot fi } + +menuentry "Network boot node with dracut: {{.Id}}" --id dracut { + initramfs="${uri}&stage=initramfs" + + uri="http://{{.Ipaddr}}:{{.Port}}/provision/${net_default_mac}?assetkey=${assetkey}" + container="${uri}&stage=container&compress=gz" + system="${uri}&stage=system&compress=gz" + runtime="${uri}&stage=runtime&compress=gz" + + {{if .KernelOverride }} + echo "Kernel: {{.KernelOverride}}" + {{else}} + echo "Kernel: {{.ContainerName}} (container default)" + {{end}} + echo "KernelArgs: {{.KernelArgs}}" + + net_args="rd.neednet=1 {{range $devname, $netdev := .NetDevs}}{{if and $netdev.Hwaddr $netdev.Device}} ifname={{$netdev.Device}}:{{$netdev.Hwaddr}} {{end}}{{end}}" + wwinit_args="root=wwinit wwinit.container=${container} wwinit.system=${system} wwinit.runtime=${runtime} init=/init" + linux $kernel wwid=${net_default_mac} {{.KernelArgs}} $net_args $wwinit_args + + if [ x$? = x0 ] ; then + echo "Loading Container: {{.ContainerName}}" + initrd $initramfs + boot + else + echo "MESSAGE: This node seems to be unconfigured. Please have your system administrator add a" + echo " configuration for this node with HW address: ${net_default_mac}" + echo "" + echo "Rebooting in 1 minute..." + sleep 60 + reboot + fi +} + menuentry "Chainload specific configfile" { conf="(http,{{.Ipaddr}}:{{.Port}})/efiboot/grub.cfg" configfile $conf } + menuentry "UEFI Firmware Settings" --id "uefi-firmware" { fwsetup } + menuentry "System restart" { echo "System rebooting..." reboot } + menuentry "System shutdown" { echo "System shutting down..." halt diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 9ef089ba..51b8d60e 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -34,6 +34,7 @@ type templateVars struct { Port string KernelArgs string KernelOverride string + Tags map[string]string NetDevs map[string]*node.NetDevs } @@ -108,7 +109,8 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { ContainerName: node.ContainerName.Get(), KernelArgs: node.Kernel.Args.Get(), KernelOverride: node.Kernel.Override.Get(), - NetDevs: tstruct.NetDevs} + NetDevs: tstruct.NetDevs, + Tags: tstruct.Tags} } else if rinfo.stage == "kernel" { if node.Kernel.Override.Defined() { stage_file = kernel.KernelImage(node.Kernel.Override.Get()) @@ -180,6 +182,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } case "grub.cfg": stage_file = path.Join(conf.Paths.Sysconfdir, "warewulf/grub/grub.cfg.ww") + tstruct := overlay.InitStruct(&node) tmpl_data = templateVars{ Id: node.Id.Get(), Cluster: node.ClusterName.Get(), @@ -190,7 +193,9 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { Hwaddr: rinfo.hwaddr, ContainerName: node.ContainerName.Get(), KernelArgs: node.Kernel.Args.Get(), - KernelOverride: node.Kernel.Override.Get()} + KernelOverride: node.Kernel.Override.Get(), + NetDevs: tstruct.NetDevs, + Tags: tstruct.Tags} if stage_file == "" { wwlog.ErrorExc(fmt.Errorf("could't find grub.cfg template"), containerName) w.WriteHeader(http.StatusNotFound) diff --git a/userdocs/contents/boot-management.rst b/userdocs/contents/boot-management.rst index d7bcec23..586a98f6 100644 --- a/userdocs/contents/boot-management.rst +++ b/userdocs/contents/boot-management.rst @@ -74,10 +74,27 @@ booting the node image directly. The iPXE template may be set at the node or profile level. +Alternatively, to direct GRUB to fetch the node's initramfs image and boot with +dracut semantics, set a ``GrubMenuEntry`` tag for the node. + +.. note:: + + Warewulf configures GRUB with a template located at + ``/etc/warewulf/grub/grub.cfg.ww``. Inspect the template to learn more about + the dracut booting process. + +.. code-block:: shell + + wwctl node set wwnode1 --tagadd GrubMenuEntry=dracut + +.. note:: + + The ``GrubMenuEntry`` variable may be set at the node or profile level. + During boot, ``warewulfd`` will detect and dynamically serve an initramfs from a node's container image in much the same way that it can serve a kernel from a container image. This image is loaded by -iPXE, and iPXE directs dracut to fetch the node's container image +iPXE (or GRUB) which directs dracut to fetch the node's container image during boot. The wwinit module provisions to tmpfs. By default, tmpfs is permitted @@ -88,7 +105,7 @@ more details.) .. warning:: - Kernel overrides are not currently supported during dracut initramfs boot. + Kernel overrides are not currently fully supported during dracut initramfs boot. Booting with GRUB =================