diff --git a/CHANGELOG.md b/CHANGELOG.md index c2d716da..41975327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Provide detected kernel version to overlay templates. #1556 - Bump github.com/containers/storage from 1.53.0 to 1.55.2 #1316, #892 - Process nodes.conf path dynamically from config. #1595, #1596, #1569 +- Split overlays into distribution and site overlays. #831 ### Removed diff --git a/Makefile b/Makefile index c027ccaf..0a94d9c9 100644 --- a/Makefile +++ b/Makefile @@ -96,9 +96,9 @@ install: build docs install -d -m 0755 $(DESTDIR)$(BINDIR) install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR) install -d -m 0755 $(DESTDIR)$(WWPROVISIONDIR) - install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR) - install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwclient/rootfs/$(WWCLIENTDIR) - install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs/$(TFTPDIR)/warewulf/ + install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR) + install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwclient/rootfs/$(WWCLIENTDIR) + install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs/$(TFTPDIR)/warewulf/ install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/examples install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/ipxe install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/grub @@ -112,27 +112,27 @@ install: build docs install -d -m 0755 $(DESTDIR)$(IPXESOURCE) install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf # wwctl genconfig to get the compiled in paths to warewulf.conf - install -d -m 0755 $(DESTDIR)$(WWDATADIR)/bmc + install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/bmc test -f $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf || ./wwctl --warewulfconf etc/warewulf.conf genconfig warewulfconf print> $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf test -f $(DESTDIR)$(WWCONFIGDIR)/nodes.conf || install -m 0644 etc/nodes.conf $(DESTDIR)$(WWCONFIGDIR) for f in etc/examples/*.ww; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/examples/; done for f in etc/ipxe/*.ipxe; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/ipxe/; done - for f in lib/warewulf/bmc/*.tmpl; do install -m 0644 $$f $(DESTDIR)$(WWDATADIR)/bmc; done + for f in lib/warewulf/bmc/*.tmpl; do install -m 0644 $$f $(DESTDIR)$(DATADIR)/warewulf/bmc; done install -m 0644 etc/grub/grub.cfg.ww $(DESTDIR)$(WWCONFIGDIR)/grub/grub.cfg.ww - install -m 0644 etc/grub/chainload.ww $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs$(TFTPDIR)/warewulf/grub.cfg.ww + install -m 0644 etc/grub/chainload.ww $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs$(TFTPDIR)/warewulf/grub.cfg.ww install -m 0644 etc/logrotate.d/warewulfd.conf $(DESTDIR)$(LOGROTATEDIR)/warewulfd.conf - (cd overlays && find * -path '*/internal' -prune -o -type f -exec install -D -m 0644 {} $(DESTDIR)$(WWOVERLAYDIR)/{} \;) - (cd overlays && find * -path '*/internal' -prune -o -type d -exec mkdir -pv $(DESTDIR)$(WWOVERLAYDIR)/{} \;) - (cd overlays && find * -path '*/internal' -prune -o -type l -exec cp -av {} $(DESTDIR)$(WWOVERLAYDIR)/{} \;) - chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/init - chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/wwinit - chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/config.ww - chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/ssh.host_keys/rootfs/etc/ssh/ssh* - chmod 0644 $(DESTDIR)$(WWOVERLAYDIR)/ssh.host_keys/rootfs/etc/ssh/ssh*.pub.ww - chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/NetworkManager/rootfs/etc/NetworkManager/system-connections/ww4-managed.ww - chmod 0750 $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs + (cd overlays && find * -path '*/internal' -prune -o -type f -exec install -D -m 0644 {} $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;) + (cd overlays && find * -path '*/internal' -prune -o -type d -exec mkdir -pv $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;) + (cd overlays && find * -path '*/internal' -prune -o -type l -exec cp -av {} $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;) + chmod 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/init + chmod 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/wwinit + chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/config.ww + chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/ssh.host_keys/rootfs/etc/ssh/ssh* + chmod 0644 $(DESTDIR)$(DATADIR)/warewulf/overlays/ssh.host_keys/rootfs/etc/ssh/ssh*.pub.ww + chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/NetworkManager/rootfs/etc/NetworkManager/system-connections/ww4-managed.ww + chmod 0750 $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs install -m 0755 wwctl $(DESTDIR)$(BINDIR) - install -m 0755 wwclient $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/wwclient + install -m 0755 wwclient $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/wwclient install -m 0644 include/firewalld/warewulf.xml $(DESTDIR)$(FIREWALLDDIR) install -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR) install -m 0644 LICENSE.md $(DESTDIR)$(WWDOCDIR) diff --git a/Variables.mk b/Variables.mk index be0f5db9..fc7c095b 100644 --- a/Variables.mk +++ b/Variables.mk @@ -61,14 +61,13 @@ endif TFTPDIR ?= /var/lib/tftpboot # Warewulf directory paths -VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR WWDATADIR IPXESOURCE +VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR IPXESOURCE WWCONFIGDIR := $(SYSCONFDIR)/$(WAREWULF) WWPROVISIONDIR := $(LOCALSTATEDIR)/$(WAREWULF)/provision WWOVERLAYDIR := $(LOCALSTATEDIR)/$(WAREWULF)/overlays WWCHROOTDIR := $(LOCALSTATEDIR)/$(WAREWULF)/chroots WWTFTPDIR := $(TFTPDIR)/$(WAREWULF) WWDOCDIR := $(DOCDIR)/$(WAREWULF) -WWDATADIR := $(DATADIR)/$(WAREWULF) WWCLIENTDIR ?= /warewulf CONFIG := $(shell pwd) diff --git a/internal/app/wwctl/node/sensors/root_test.go b/internal/app/wwctl/node/sensors/root_test.go index 450aab4c..a2058130 100644 --- a/internal/app/wwctl/node/sensors/root_test.go +++ b/internal/app/wwctl/node/sensors/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/overlay/chmod/main.go b/internal/app/wwctl/overlay/chmod/main.go index 22a92cf7..3a7f28d6 100644 --- a/internal/app/wwctl/overlay/chmod/main.go +++ b/internal/app/wwctl/overlay/chmod/main.go @@ -3,7 +3,6 @@ package chmod import ( "fmt" "os" - "path" "strconv" "github.com/spf13/cobra" @@ -12,8 +11,6 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var overlaySourceDir string - overlayName := args[0] fileName := args[1] @@ -21,16 +18,19 @@ func CobraRunE(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("could not convert requested mode: %s", err) } - - overlaySourceDir = overlay.OverlaySourceDir(overlayName) - - if !util.IsDir(overlaySourceDir) { + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.IsSiteOverlay() { + overlay_, err = overlay_.CloneSiteOverlay() + if err != nil { + return err + } + } + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } - overlayFile := path.Join(overlaySourceDir, fileName) - - if !util.IsFile(overlayFile) && !util.IsDir(overlayFile) { + overlayFile := overlay_.File(fileName) + if !(util.IsFile(overlayFile) || util.IsDir(overlayFile)) { return fmt.Errorf("file does not exist within overlay: %s:%s", overlayName, fileName) } diff --git a/internal/app/wwctl/overlay/chown/main.go b/internal/app/wwctl/overlay/chown/main.go index 10d82a96..2afa2068 100644 --- a/internal/app/wwctl/overlay/chown/main.go +++ b/internal/app/wwctl/overlay/chown/main.go @@ -3,7 +3,6 @@ package chown import ( "fmt" "os" - "path" "strconv" "github.com/warewulf/warewulf/internal/pkg/overlay" @@ -13,7 +12,6 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var overlaySourceDir string var uid int var gid int var err error @@ -35,15 +33,20 @@ func CobraRunE(cmd *cobra.Command, args []string) error { gid = -1 } - overlaySourceDir = overlay.OverlaySourceDir(overlayName) - - if !util.IsDir(overlaySourceDir) { + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } - overlayFile := path.Join(overlaySourceDir, fileName) + if !overlay_.IsSiteOverlay() { + overlay_, err = overlay_.CloneSiteOverlay() + if err != nil { + return err + } + } - if !util.IsFile(overlayFile) && !util.IsDir(overlayFile) { + overlayFile := overlay_.File(fileName) + if !(util.IsFile(overlayFile) || util.IsDir(overlayFile)) { return fmt.Errorf("file does not exist within overlay: %s:%s", overlayName, fileName) } diff --git a/internal/app/wwctl/overlay/create/main.go b/internal/app/wwctl/overlay/create/main.go index 966f43f7..201c02f2 100644 --- a/internal/app/wwctl/overlay/create/main.go +++ b/internal/app/wwctl/overlay/create/main.go @@ -6,6 +6,5 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) (err error) { - err = overlay.OverlayInit(args[0]) - return + return overlay.GetSiteOverlay(args[0]).Create() } diff --git a/internal/app/wwctl/overlay/delete/main.go b/internal/app/wwctl/overlay/delete/main.go index 3ceb26b7..ee439c08 100644 --- a/internal/app/wwctl/overlay/delete/main.go +++ b/internal/app/wwctl/overlay/delete/main.go @@ -5,7 +5,6 @@ import ( "os" "path" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/overlay" "github.com/warewulf/warewulf/internal/pkg/util" @@ -13,7 +12,6 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var overlayPath string var fileName string overlayName := args[0] @@ -22,27 +20,22 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fileName = args[1] } - overlayPath = overlay.OverlaySourceDir(overlayName) - - if overlayPath == "" { - return fmt.Errorf("overlay name did not resolve: '%s'", overlayName) + overlay_ := overlay.GetOverlay(overlayName) + if overlay_.IsDistributionOverlay() { + return fmt.Errorf("distribution overlay can't deleted") } - - if !util.IsDir(overlayPath) { + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } if fileName == "" { - if overlayName == "wwinit" || overlayName == "host" { - return errors.New("refusing to delete the Warewulf overlay") - } if Force { - err := os.RemoveAll(overlayPath) + err := os.RemoveAll(overlay_.Path()) if err != nil { return fmt.Errorf("failed deleting overlay: %w", err) } } else { - err := os.Remove(overlayPath) + err := os.Remove(overlay_.Path()) if err != nil { return fmt.Errorf("failed deleting overlay: %w", err) } @@ -50,28 +43,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Info("Deleted overlay: %s\n", args[0]) } else { - removePath := path.Join(overlayPath, fileName) + removePath := overlay_.File(fileName) - if !util.IsDir(removePath) && !util.IsFile(removePath) { + if !(util.IsDir(removePath) || util.IsFile(removePath)) { return fmt.Errorf("path to remove doesn't exist in overlay: %s", removePath) } if Force { err := os.RemoveAll(removePath) if err != nil { - return fmt.Errorf("failed deleting file from overlay: %s:%s", overlayName, overlayPath) + return fmt.Errorf("failed deleting file from overlay: %s:%s", overlayName, removePath) } } else { err := os.Remove(removePath) if err != nil { - return fmt.Errorf("failed deleting overlay: %s:%s", overlayName, overlayPath) + return fmt.Errorf("failed deleting overlay: %s:%s", overlayName, removePath) } } if Parents { // Cleanup any empty directories left behind... i := path.Dir(removePath) - for i != overlayPath { + for i != overlay_.Rootfs() { wwlog.Debug("Evaluating directory to remove: %s", i) err := os.Remove(i) if err != nil { diff --git a/internal/app/wwctl/overlay/edit/main.go b/internal/app/wwctl/overlay/edit/main.go index b8a1b820..bb54d563 100644 --- a/internal/app/wwctl/overlay/edit/main.go +++ b/internal/app/wwctl/overlay/edit/main.go @@ -30,33 +30,22 @@ const initialTemplate = `# This is a Warewulf Template file. # Keep the following for better reference: # --- # This file is autogenerated by warewulf -# Host: {{.BuildHost}} -# Time: {{.BuildTime}} -# Source: {{.BuildSource}} ` -func CobraRunE(cmd *cobra.Command, args []string) error { +func CobraRunE(cmd *cobra.Command, args []string) (err error) { overlayName := args[0] fileName := args[1] - overlaySourceDir := overlay.OverlaySourceDir(overlayName) - if !util.IsDir(overlaySourceDir) { + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } - overlayFile := path.Join(overlaySourceDir, fileName) + overlayFile := overlay_.File(fileName) wwlog.Debug("Will edit overlay file: %s", overlayFile) - overlayFileDir := path.Dir(overlayFile) - if CreateDirs { - err := os.MkdirAll(overlayFileDir, 0755) - if err != nil { - return fmt.Errorf("could not create directory: %s", overlayFileDir) - } - } else { - if !util.IsDir(overlayFileDir) { - return fmt.Errorf("%s does not exist. Use '--parents' option to create automatically", overlayFileDir) - } + if !(util.IsDir(overlayFileDir) || CreateDirs) { + return fmt.Errorf("%s does not exist. Use '--parents' option to create automatically", overlayFileDir) } tempFile, tempFileErr := os.CreateTemp("", "ww-overlay-edit-") @@ -102,12 +91,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } else { if startTime == fileInfo.ModTime() { wwlog.Debug("No change detected. Not updating overlay.") - os.Exit(0) + return nil + } + } + + if !overlay_.IsSiteOverlay() { + overlay_, err = overlay_.CloneSiteOverlay() + if err != nil { + return err + } + } + // re-generate because overlay_ may have changed + overlayFile = overlay_.File(fileName) + overlayFileDir = path.Dir(overlayFile) + + if CreateDirs { + if err := os.MkdirAll(overlayFileDir, 0755); err != nil { + return fmt.Errorf("could not create directory: %s", overlayFileDir) } } // try renaming the tempfile to overlayfile first - err := os.Rename(tempFile.Name(), overlayFile) + err = os.Rename(tempFile.Name(), overlayFile) if err != nil { // if it fails, which probably means that they exists on different partitions // fallback to data copy diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index acad75ec..8a890bb1 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -13,9 +13,8 @@ import ( "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func CobraRunE(cmd *cobra.Command, args []string) error { +func CobraRunE(cmd *cobra.Command, args []string) (err error) { var dest string - var overlaySource string overlayName := args[0] source := args[1] @@ -27,23 +26,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } wwlog.Verbose("Copying '%s' into overlay '%s:%s'", source, overlayName, dest) - overlaySource = overlay.OverlaySourceDir(overlayName) - - if !util.IsDir(overlaySource) { + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.IsSiteOverlay() { + overlay_, err = overlay_.CloneSiteOverlay() + if err != nil { + return err + } + } + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } - if util.IsDir(path.Join(overlaySource, dest)) { + if util.IsDir(overlay_.File(dest)) { dest = path.Join(dest, path.Base(source)) } - if util.IsFile(path.Join(overlaySource, dest)) { + if util.IsFile(overlay_.File(dest)) { return fmt.Errorf("a file with that name already exists in the overlay: %s", overlayName) } if CreateDirs { - parent := filepath.Dir(path.Join(overlaySource, dest)) - if _, err := os.Stat(parent); os.IsNotExist(err) { + parent := filepath.Dir(overlay_.File(dest)) + if _, err = os.Stat(parent); os.IsNotExist(err) { wwlog.Debug("Create dir: %s", parent) srcInfo, err := os.Stat(source) if err != nil { @@ -56,7 +60,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } - err := util.CopyFile(source, path.Join(overlaySource, dest)) + err = util.CopyFile(source, overlay_.File(dest)) if err != nil { return fmt.Errorf("could not copy file into overlay: %w", err) } diff --git a/internal/app/wwctl/overlay/list/main.go b/internal/app/wwctl/overlay/list/main.go index c5ad2040..d27cfcb3 100644 --- a/internal/app/wwctl/overlay/list/main.go +++ b/internal/app/wwctl/overlay/list/main.go @@ -6,6 +6,7 @@ import ( "syscall" "github.com/spf13/cobra" + "github.com/warewulf/warewulf/internal/app/wwctl/table" "github.com/warewulf/warewulf/internal/pkg/overlay" "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" @@ -24,51 +25,52 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } + t := table.New(cmd.OutOrStdout()) if ListLong { - wwlog.Info("%-10s %5s %-5s %-18s %s\n", "PERM MODE", "UID", "GID", "SYSTEM-OVERLAY", "FILE PATH") + t.AddHeader("PERM MODE", "UID", "GID", "SYSTEM-OVERLAY", "FILE PATH", "SITE") } else { - wwlog.Info("%-30s %-12s\n", "OVERLAY NAME", "FILES/DIRS") + t.AddHeader("OVERLAY NAME", "FILES/DIRS", "SITE") } - for o := range overlays { - name := overlays[o] - path := overlay.OverlaySourceDir(name) + for _, name := range overlays { + overlay_ := overlay.GetOverlay(name) - if util.IsDir(path) { - files := util.FindFiles(path) + if !overlay_.Exists() { + wwlog.Error("system/%s (path not found:%s)", name, overlay_.Rootfs()) + continue + } - wwlog.Debug("Iterating overlay path: %s", path) - if ListLong { - for file := range files { - s, err := os.Stat(files[file]) - if err != nil { - continue - } + files := util.FindFiles(overlay_.Rootfs()) - fileMode := s.Mode() - perms := fileMode & os.ModePerm - - sys := s.Sys() - - wwlog.Info("%v %5d %-5d %-18s /%s\n", perms, sys.(*syscall.Stat_t).Uid, sys.(*syscall.Stat_t).Gid, overlays[o], files[file]) + wwlog.Debug("Iterating overlay rootfs: %s", overlay_.Rootfs()) + if ListLong { + for file := range files { + s, err := os.Stat(files[file]) + if err != nil { + continue } - } else if ListContents { - var fileCount int - for file := range files { - wwlog.Info("%-30s /%-12s\n", name, files[file]) - fileCount++ - } - if fileCount == 0 { - wwlog.Info("%-30s %-12d\n", name, 0) - } - } else { - wwlog.Info("%-30s %-12d\n", name, len(files)) + + fileMode := s.Mode() + perms := fileMode & os.ModePerm + + sys := s.Sys() + + t.AddLine(perms, sys.(*syscall.Stat_t).Uid, sys.(*syscall.Stat_t).Gid, name, files[file], overlay_.IsSiteOverlay()) + } + } else if ListContents { + var fileCount int + for file := range files { + t.AddLine(name, files[file], overlay_.IsSiteOverlay()) + fileCount++ + } + if fileCount == 0 { + t.AddLine(name, 0, overlay_.IsSiteOverlay()) } - } else { - wwlog.Error("system/%s (path not found:%s)", overlays[o], path) + t.AddLine(name, len(files), overlay_.IsSiteOverlay()) } } + t.Print() return nil } diff --git a/internal/app/wwctl/overlay/list/main_test.go b/internal/app/wwctl/overlay/list/main_test.go index b8d9c542..05cd4257 100644 --- a/internal/app/wwctl/overlay/list/main_test.go +++ b/internal/app/wwctl/overlay/list/main_test.go @@ -2,7 +2,6 @@ package list import ( "bytes" - "path" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +13,7 @@ import ( func Test_Overlay_List(t *testing.T) { env := testenv.New(t) - env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), ` + env.WriteFile(t, "var/lib/warewulf/overlays/testoverlay/email.ww", ` {{ if .Tags.email }}eMail: {{ .Tags.email }}{{else}} noMail{{- end }} `) defer env.RemoveAll(t) diff --git a/internal/app/wwctl/overlay/mkdir/main.go b/internal/app/wwctl/overlay/mkdir/main.go index ee8ffd8c..9314fabb 100644 --- a/internal/app/wwctl/overlay/mkdir/main.go +++ b/internal/app/wwctl/overlay/mkdir/main.go @@ -3,33 +3,33 @@ package mkdir import ( "fmt" "os" - "path" "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/overlay" - "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func CobraRunE(cmd *cobra.Command, args []string) error { - var overlaySourceDir string - +func CobraRunE(cmd *cobra.Command, args []string) (err error) { overlayName := args[0] dirName := args[1] - overlaySourceDir = overlay.OverlaySourceDir(overlayName) + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.IsSiteOverlay() { + overlay_, err = overlay_.CloneSiteOverlay() + if err != nil { + return err + } + } - if !util.IsDir(overlaySourceDir) { + if !overlay_.Exists() { return fmt.Errorf("overlay does not exist: %s", overlayName) } - overlayDir := path.Join(overlaySourceDir, dirName) - + overlayDir := overlay_.File(dirName) wwlog.Debug("Will create directory in overlay: %s:%s", overlayName, dirName) - - err := os.MkdirAll(overlayDir, os.FileMode(PermMode)) + err = os.MkdirAll(overlayDir, os.FileMode(PermMode)) if err != nil { - return fmt.Errorf("could not create directory: %s", path.Dir(overlayDir)) + return fmt.Errorf("could not create directory: %s", overlayDir) } return nil diff --git a/internal/app/wwctl/overlay/show/main.go b/internal/app/wwctl/overlay/show/main.go index 198a2099..97f7a3e4 100644 --- a/internal/app/wwctl/overlay/show/main.go +++ b/internal/app/wwctl/overlay/show/main.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "os" - "path" "path/filepath" "regexp" "strings" @@ -18,18 +17,15 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var overlaySourceDir string - overlayName := args[0] fileName := args[1] - overlaySourceDir = overlay.OverlaySourceDir(overlayName) - if !util.IsDir(overlaySourceDir) { - return fmt.Errorf("overlay dir: %s does not exist", overlaySourceDir) + overlay_ := overlay.GetOverlay(overlayName) + if !overlay_.Exists() { + return fmt.Errorf("overlay does not exist: %s", overlayName) } - overlayFile := path.Join(overlaySourceDir, fileName) - + overlayFile := overlay_.File(fileName) if !util.IsFile(overlayFile) { return fmt.Errorf("file: %s does not exist within overlay", overlayFile) } @@ -61,7 +57,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodeConf = node.NewNode(hostName) nodeConf.ClusterName = hostName } - tstruct, err := overlay.InitStruct(nodeConf) + tstruct, err := overlay.InitStruct(overlayName, nodeConf) if err != nil { return err } diff --git a/internal/app/wwctl/overlay/show/main_test.go b/internal/app/wwctl/overlay/show/main_test.go index e341b644..1f67bed5 100644 --- a/internal/app/wwctl/overlay/show/main_test.go +++ b/internal/app/wwctl/overlay/show/main_test.go @@ -69,8 +69,10 @@ nodes: - empty `) - env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), overlayEmail) - env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/overlay.ww"), overlayOverlay) + env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/email.ww", overlayEmail) + env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/overlay.ww", overlayOverlay) + env.WriteFile(t, "usr/share/warewulf/overlays/dist/foo.ww", "foo") + env.WriteFile(t, "var/lib/warewulf/overlays/dist/foo.ww", "foobaar") defer env.RemoveAll(t) warewulfd.SetNoDaemon() t.Run("overlay show raw", func(t *testing.T) { @@ -128,6 +130,17 @@ nodes: assert.NoError(t, err) assert.Contains(t, buf.String(), "testoverlay") }) + t.Run("site overlays precede", func(t *testing.T) { + baseCmd.SetArgs([]string{"-r", "node1", "dist", "foo.ww"}) + baseCmd := GetCommand() + buf := new(bytes.Buffer) + baseCmd.SetOut(buf) + baseCmd.SetErr(buf) + wwlog.SetLogWriter(buf) + err := baseCmd.Execute() + assert.NoError(t, err) + assert.Contains(t, buf.String(), "foobaar") + }) } func TestShowServerTemplate(t *testing.T) { diff --git a/internal/app/wwctl/power/cycle/root_test.go b/internal/app/wwctl/power/cycle/root_test.go index 9e20db8f..a6b0d392 100644 --- a/internal/app/wwctl/power/cycle/root_test.go +++ b/internal/app/wwctl/power/cycle/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/power/off/root_test.go b/internal/app/wwctl/power/off/root_test.go index 5dc368c8..a926a0cf 100644 --- a/internal/app/wwctl/power/off/root_test.go +++ b/internal/app/wwctl/power/off/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/power/on/root_test.go b/internal/app/wwctl/power/on/root_test.go index 8ea62f84..c1aa3d78 100644 --- a/internal/app/wwctl/power/on/root_test.go +++ b/internal/app/wwctl/power/on/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/power/reset/root_test.go b/internal/app/wwctl/power/reset/root_test.go index 8c9a0629..b71a0d34 100644 --- a/internal/app/wwctl/power/reset/root_test.go +++ b/internal/app/wwctl/power/reset/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/power/soft/root_test.go b/internal/app/wwctl/power/soft/root_test.go index 565f2d07..5aed7dbc 100644 --- a/internal/app/wwctl/power/soft/root_test.go +++ b/internal/app/wwctl/power/soft/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/app/wwctl/power/status/root_test.go b/internal/app/wwctl/power/status/root_test.go index 998dee14..2045c7a9 100644 --- a/internal/app/wwctl/power/status/root_test.go +++ b/internal/app/wwctl/power/status/root_test.go @@ -28,7 +28,7 @@ nodes: - default ipmi: ipaddr: 10.10.10.10`) - env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") + env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl") tests := map[string]struct { args []string diff --git a/internal/pkg/config/buildconfig.go.in b/internal/pkg/config/buildconfig.go.in index f0e6c31e..45dd5969 100644 --- a/internal/pkg/config/buildconfig.go.in +++ b/internal/pkg/config/buildconfig.go.in @@ -9,18 +9,19 @@ import ( var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf" type BuildConfig struct { - Bindir string `default:"@BINDIR@"` - Sysconfdir string `default:"@SYSCONFDIR@"` - Localstatedir string `default:"@LOCALSTATEDIR@"` - Cachedir string `default:"@CACHEDIR@"` - Ipxesource string `default:"@IPXESOURCE@"` - Srvdir string `default:"@SRVDIR@"` - Firewallddir string `default:"@FIREWALLDDIR@"` - Systemddir string `default:"@SYSTEMDDIR@"` - WWOverlaydir string `default:"@WWOVERLAYDIR@"` - WWChrootdir string `default:"@WWCHROOTDIR@"` - WWProvisiondir string `default:"@WWPROVISIONDIR@"` - WWClientdir string `default:"@WWCLIENTDIR@"` + Bindir string `yaml:"bindir,omitempty" default:"@BINDIR@"` + Sysconfdir string `yaml:"sysconfdir,omitempty" default:"@SYSCONFDIR@"` + Localstatedir string `yaml:"localstatedir,omitempty" default:"@LOCALSTATEDIR@"` + Cachedir string `yaml:"cachedir,omitempty" default:"@CACHEDIR@"` + Ipxesource string `yaml:"ipxesource,omitempty" default:"@IPXESOURCE@"` + Srvdir string `yaml:"srvdir,omitempty" default:"@SRVDIR@"` + Firewallddir string `yaml:"firewallddir,omitempty" default:"@FIREWALLDDIR@"` + Systemddir string `yaml:"systemddir,omitempty" default:"@SYSTEMDDIR@"` + Datadir string `yaml:"datadir,omitempty" default:"@DATADIR@"` + WWOverlaydir string `yaml:"wwoverlaydir,omitempty" default:"@WWOVERLAYDIR@"` + WWChrootdir string `yaml:"wwchrootdir,omitempty" default:"@WWCHROOTDIR@"` + WWProvisiondir string `yaml:"wwprovisiondir,omitempty" default:"@WWPROVISIONDIR@"` + WWClientdir string `yaml:"wwclientdir,omitempty" default:"@WWCLIENTDIR@"` } const Version = "@VERSION@" @@ -41,14 +42,13 @@ func (this TFTPConf) Enabled() bool { // WarewulfConf adds additional Warewulf-specific configuration to // BaseConf. type WarewulfConf struct { - Port int `yaml:"port,omitempty" default:"9873"` - SecureP *bool `yaml:"secure,omitempty" default:"true"` - UpdateInterval int `yaml:"update interval,omitempty" default:"60"` - AutobuildOverlaysP *bool `yaml:"autobuild overlays,omitempty" default:"true"` - EnableHostOverlayP *bool `yaml:"host overlay,omitempty" default:"true"` - SyslogP *bool `yaml:"syslog,omitempty" default:"false"` - DataStore string `yaml:"datastore,omitempty" default:"@DATADIR@"` - GrubBootP *bool `yaml:"grubboot,omitempty" default:"false"` + Port int `yaml:"port,omitempty" default:"9873"` + SecureP *bool `yaml:"secure,omitempty" default:"true"` + UpdateInterval int `yaml:"update interval,omitempty" default:"60"` + AutobuildOverlaysP *bool `yaml:"autobuild overlays,omitempty" default:"true"` + EnableHostOverlayP *bool `yaml:"host overlay,omitempty" default:"true"` + SyslogP *bool `yaml:"syslog,omitempty" default:"false"` + GrubBootP *bool `yaml:"grubboot,omitempty" default:"false"` } func (this WarewulfConf) Secure() bool { @@ -79,6 +79,14 @@ func (paths BuildConfig) OciBlobCachedir() string { return path.Join(paths.Cachedir, "warewulf") } +func (paths BuildConfig) SiteOverlaydir() string { + return paths.WWOverlaydir +} + +func (paths BuildConfig) DistributionOverlaydir() string { + return path.Join(paths.Datadir, "warewulf", "overlays") +} + func (paths BuildConfig) OverlayProvisiondir() string { return path.Join(paths.WWProvisiondir, "overlays") } diff --git a/internal/pkg/configure/hostfile.go b/internal/pkg/configure/hostfile.go index bb7d469b..3046eb41 100644 --- a/internal/pkg/configure/hostfile.go +++ b/internal/pkg/configure/hostfile.go @@ -14,13 +14,14 @@ import ( Creates '/etc/hosts' from the host template. */ func Hostfile() (err error) { - hostTemplate := path.Join(overlay.OverlaySourceDir("host"), "/etc/hosts.ww") + overlay_ := overlay.GetOverlay("host") + hostTemplate := path.Join(overlay_.Rootfs(), "/etc/hosts.ww") if !(util.IsFile(hostTemplate)) { return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay") } hostname, _ := os.Hostname() - tstruct, err := overlay.InitStruct(node.NewNode(hostname)) + tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname)) if err != nil { return err } diff --git a/internal/pkg/oci/defaults.go b/internal/pkg/oci/defaults.go index 24643e5c..e3a516bf 100644 --- a/internal/pkg/oci/defaults.go +++ b/internal/pkg/oci/defaults.go @@ -6,7 +6,7 @@ import ( warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" ) -var defaultCachePath = filepath.Join(warewulfconf.Get().Warewulf.DataStore, "/container-cache/oci/") +var defaultCachePath = filepath.Join(warewulfconf.Get().Paths.Datadir, "/container-cache/oci/") const ( blobPrefix = "blobs" diff --git a/internal/pkg/overlay/config.go b/internal/pkg/overlay/config.go index 988519a3..5d15b43e 100644 --- a/internal/pkg/overlay/config.go +++ b/internal/pkg/overlay/config.go @@ -1,31 +1,74 @@ package overlay import ( + "fmt" "os" "path" "strings" - warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" + "github.com/containers/storage/drivers/copy" + + "github.com/warewulf/warewulf/internal/pkg/config" + "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func OverlaySourceTopDir() string { - conf := warewulfconf.Get() - return conf.Paths.WWOverlaydir +// GetOverlay returns the filesystem path of an overlay identified by its name, +// along with a boolean indicating whether the returned overlayPath corresponds +// to a site-specific overlay. +func GetOverlay(name string) (overlay Overlay) { + overlay = GetSiteOverlay(name) + if overlay.Exists() { + return overlay + } + overlay = GetDistributionOverlay(name) + if overlay.Exists() { + return overlay + } + return GetSiteOverlay(name) } -/* -Return the path for the base of the overlay, strips rootfs -prefix in the overlay dir if this it exists -*/ -func OverlaySourceDir(overlayName string) string { - /* Assume using old style overlay dir without rootfs */ - var overlaypath = path.Join(OverlaySourceTopDir(), overlayName) - if _, err := os.Stat(path.Join(overlaypath, "rootfs")); err == nil { - /* rootfs exists, use it. */ - overlaypath = path.Join(overlaypath, "rootfs") +// GetDistributionOverlay returns the filesystem path of a distribution overlay +// identified by the given name. +func GetDistributionOverlay(name string) (overlay Overlay) { + return getOverlay(config.Get().Paths.DistributionOverlaydir(), name) +} + +// GetSiteOverlay returns the filesystem path of a site-specific overlay +// identified by the given name. +func GetSiteOverlay(name string) (overlay Overlay) { + return getOverlay(config.Get().Paths.SiteOverlaydir(), name) +} + +// getOverlay constructs an overlay based on the given overlay directory and +// overlay name. The overlay does not necessarily exist. +func getOverlay(overlaydir, name string) (overlay Overlay) { + return Overlay(path.Join(overlaydir, name)) +} + +// Create creates a new overlay directory for the given overlay +// +// Returns an error if the overlay already exists or if directory creation fails. +func (this Overlay) Create() error { + if util.IsDir(this.Path()) { + return fmt.Errorf("overlay already exists: %s", this) } - return overlaypath + return os.MkdirAll(this.Rootfs(), 0755) +} + +// Creates a site overlay from an existing distribution overlay. +// +// If the distribution overlay doesn't exist, return an error. +func (this Overlay) CloneSiteOverlay() (siteOverlay Overlay, err error) { + siteOverlay = GetSiteOverlay(this.Name()) + if !util.IsDir(this.Path()) { + return siteOverlay, fmt.Errorf("source overlay does not exist: %s", this.Name()) + } + if siteOverlay.Exists() { + return siteOverlay, fmt.Errorf("site overlay already exists: %s", siteOverlay.Name()) + } + err = copy.DirCopy(this.Path(), siteOverlay.Path(), copy.Content, true) + return siteOverlay, err } // OverlayImage returns the full path to an overlay image based on the @@ -54,6 +97,5 @@ func OverlayImage(nodeName string, context string, overlayNames []string) string return "" } - conf := warewulfconf.Get() - return path.Join(conf.Paths.OverlayProvisiondir(), nodeName, name) + return path.Join(config.Get().Paths.OverlayProvisiondir(), nodeName, name) } diff --git a/internal/pkg/overlay/datastructure.go b/internal/pkg/overlay/datastructure.go index d2c71d5b..67c61d5f 100644 --- a/internal/pkg/overlay/datastructure.go +++ b/internal/pkg/overlay/datastructure.go @@ -46,8 +46,9 @@ type TemplateStruct struct { /* Initialize an TemplateStruct with the given node.NodeInfo */ -func InitStruct(nodeData node.Node) (TemplateStruct, error) { +func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) { var tstruct TemplateStruct + tstruct.Overlay = overlayName hostname, _ := os.Hostname() tstruct.BuildHost = hostname controller := warewulfconf.Get() diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index bef2edc3..5628d004 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -14,8 +14,9 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/pkg/errors" + + "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" @@ -25,6 +26,82 @@ var ( ErrDoesNotExist = errors.New("overlay does not exist") ) +// Overlay represents an overlay directory path. +type Overlay string + +// Name returns the base name of the overlay directory. +// +// This is derived from the full path of the overlay. +func (this Overlay) Name() string { + return path.Base(this.Path()) +} + +// Path returns the string representation of the overlay path. +// +// This method allows the Overlay type to be easily converted back to its +// underlying string representation. +func (this Overlay) Path() string { + return string(this) +} + +// Rootfs returns the path to the root filesystem (rootfs) within the overlay. +// +// If the "rootfs" directory exists inside the overlay path, it returns the +// path to the "rootfs" directory. Otherwise, it checks if the overlay path +// itself is a directory and returns that. If neither exists, it defaults to +// returning the "rootfs" path. +func (this Overlay) Rootfs() string { + rootfs := path.Join(this.Path(), "rootfs") + if util.IsDir(rootfs) { + return rootfs + } else if util.IsDir(this.Path()) { + return this.Path() + } else { + return rootfs + } +} + +// File constructs a full path to a file within the overlay's root filesystem. +// +// Parameters: +// - filePath: The relative path of the file within the overlay. +// +// Returns: +// - The full path to the specified file in the overlay's rootfs. +func (this Overlay) File(filePath string) string { + return path.Join(this.Rootfs(), filePath) +} + +// Exists checks whether the overlay path exists and is a directory. +// +// Returns: +// - true if the overlay path exists and is a directory; false otherwise. +func (this Overlay) Exists() bool { + return util.IsDir(this.Path()) +} + +// IsSiteOverlay determines whether the overlay is a site overlay. +// +// A site overlay is identified by its parent directory matching the configured +// site overlay directory path. +// +// Returns: +// - true if the overlay is a site overlay; false otherwise. +func (this Overlay) IsSiteOverlay() bool { + return path.Dir(this.Path()) == config.Get().Paths.SiteOverlaydir() +} + +// IsDistributionOverlay determines whether the overlay is a distribution overlay. +// +// A distribution overlay is identified by its parent directory matching the configured +// distribution overlay directory path. +// +// Returns: +// - true if the overlay is a distribution overlay; false otherwise. +func (this Overlay) IsDistributionOverlay() bool { + return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() +} + /* Build all overlays for a node */ @@ -70,7 +147,7 @@ func BuildHostOverlay() error { hostname, _ := os.Hostname() hostData := node.NewNode(hostname) wwlog.Info("Building overlay for %s: host", hostname) - hostdir := OverlaySourceDir("host") + hostdir := GetOverlay("host").Rootfs() stats, err := os.Stat(hostdir) if err != nil { return fmt.Errorf("could not build host overlay: %w ", err) @@ -84,40 +161,26 @@ func BuildHostOverlay() error { /* Get all overlays present in warewulf */ -func FindOverlays() ([]string, error) { - var ret []string +func FindOverlays() (overlayList []string, err error) { dotfilecheck, _ := regexp.Compile(`^\..*`) - - files, err := os.ReadDir(OverlaySourceTopDir()) - if err != nil { - return ret, fmt.Errorf("could not get list of overlays: %w", err) + controller := config.Get() + var files []fs.DirEntry + if distfiles, err := os.ReadDir(controller.Paths.DistributionOverlaydir()); err == nil { + files = append(files, distfiles...) + } + if sitefiles, err := os.ReadDir(path.Join(controller.Paths.SiteOverlaydir())); err == nil { + files = append(files, sitefiles...) } - for _, file := range files { wwlog.Debug("Evaluating overlay source: %s", file.Name()) isdotfile := dotfilecheck.MatchString(file.Name()) if (file.IsDir()) && !(isdotfile) { - ret = append(ret, file.Name()) + overlayList = append(overlayList, file.Name()) } } - return ret, nil -} - -/* -Creates an empty overlay -*/ -func OverlayInit(overlayName string) error { - path := OverlaySourceDir(overlayName) - - if util.IsDir(path) { - return errors.New("Overlay already exists: " + overlayName) - } - - err := os.MkdirAll(path, 0755) - - return err + return overlayList, nil } /* @@ -190,7 +253,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri wwlog.Verbose("Processing node/overlay: %s/%s", nodeData.Id(), strings.Join(overlayNames, "-")) for _, overlayName := range overlayNames { wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir) - overlaySourceDir := OverlaySourceDir(overlayName) + overlaySourceDir := GetOverlay(overlayName).Rootfs() wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir) err := os.Chdir(overlaySourceDir) if err != nil { @@ -220,7 +283,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri wwlog.Debug("Created directory in overlay: %s", location) } else if filepath.Ext(location) == ".ww" { - tstruct, err := InitStruct(nodeData) + tstruct, err := InitStruct(overlayName, nodeData) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } @@ -358,19 +421,6 @@ func RenderTemplateFile(fileName string, data TemplateStruct) ( err error) { backupFile = true writeFile = true - // parse the overlay name out of the absolute path - overlayPath, _ := filepath.Rel(OverlaySourceTopDir(), fileName) - withoutRootfs, err := filepath.Rel("rootfs", overlayPath) - if err != nil { - overlayPath = withoutRootfs - } - overlayPathElem := strings.Split(overlayPath, "/") - if len(overlayPathElem) > 1 { - data.Overlay = overlayPathElem[0] - } else { - data.Overlay = "" - } - // Build our FuncMap funcMap := template.FuncMap{ "Include": templateFileInclude, @@ -442,7 +492,7 @@ func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { // Get all the files as a string slice for a given overlay func OverlayGetFiles(name string) (files []string, err error) { - baseDir := OverlaySourceDir(name) + baseDir := GetOverlay(name).Rootfs() if !util.IsDir(baseDir) { err = fmt.Errorf("overlay %s doesn't exist", name) return diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 56dab2f4..a409041d 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -10,10 +10,106 @@ import ( "github.com/stretchr/testify/assert" warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" + "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/util" ) +func Test_OverlayMethods(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + + // Setup test data + sitedir := "/var/lib/warewulf/overlays/" + distdir := "/usr/share/warewulf/overlays/" + + env.WriteFile(t, path.Join(sitedir, "siteonly/rootfs/testfile"), "a site overlay") + env.WriteFile(t, path.Join(distdir, "distonly/rootfs/testfile"), "a distribution overlay") + env.WriteFile(t, path.Join(sitedir, "legacy/testfile"), "a legacy overlay") + env.WriteFile(t, path.Join(sitedir, "both/rootfs/testfile"), "the site version") + env.WriteFile(t, path.Join(distdir, "both/rootfs/testfile"), "the distribution version") + + var tests = map[string]struct { + name string + path string + rootfs string + file string + content string + exists bool + isSite bool + isDist bool + }{ + "site overlay": { + name: "siteonly", + path: path.Join(sitedir, "siteonly"), + rootfs: path.Join(sitedir, "siteonly/rootfs"), + file: path.Join(sitedir, "siteonly/rootfs/testfile"), + content: "a site overlay", + exists: true, + isSite: true, + isDist: false, + }, + "distribution overlay": { + name: "distonly", + path: path.Join(distdir, "distonly"), + rootfs: path.Join(distdir, "distonly/rootfs"), + file: path.Join(distdir, "distonly/rootfs/testfile"), + content: "a distribution overlay", + exists: true, + isSite: false, + isDist: true, + }, + "overlapping overlay": { + name: "both", + path: path.Join(sitedir, "both"), + rootfs: path.Join(sitedir, "both/rootfs"), + file: path.Join(sitedir, "both/rootfs/testfile"), + content: "the site version", + exists: true, + isSite: true, + isDist: false, + }, + "legacy overlay": { + name: "legacy", + path: path.Join(sitedir, "legacy"), + rootfs: path.Join(sitedir, "legacy"), + file: path.Join(sitedir, "legacy/testfile"), + content: "", + exists: true, + isSite: true, + isDist: false, + }, + "missing overlay": { + name: "absent", + path: path.Join(sitedir, "absent"), + rootfs: path.Join(sitedir, "absent/rootfs"), + file: path.Join(sitedir, "absent/rootfs/testfile"), + content: "", + exists: false, + isSite: true, + isDist: false, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + overlay := GetOverlay(tt.name) + assert.Equal(t, tt.name, overlay.Name()) + assert.Equal(t, env.GetPath(tt.path), overlay.Path()) + assert.Equal(t, env.GetPath(tt.rootfs), overlay.Rootfs()) + assert.Equal(t, env.GetPath(tt.file), overlay.File("testfile")) + if tt.content != "" { + buffer, err := os.ReadFile(overlay.File("testfile")) + assert.NoError(t, err) + assert.Equal(t, tt.content, string(buffer)) + } + assert.Equal(t, tt.exists, overlay.Exists()) + assert.Equal(t, tt.isSite, overlay.IsSiteOverlay()) + assert.Equal(t, tt.isDist, overlay.IsDistributionOverlay()) + }) + } +} + var buildOverlayTests = []struct { description string nodeName string diff --git a/internal/pkg/power/ipmitool.go b/internal/pkg/power/ipmitool.go index dc543769..8e54eab0 100644 --- a/internal/pkg/power/ipmitool.go +++ b/internal/pkg/power/ipmitool.go @@ -37,7 +37,7 @@ func (ipmi *IPMI) getStr() (cmdStr string, err error) { } if !strings.HasPrefix(ipmi.Template, "/") { conf := warewulfconf.Get() - ipmi.Template = path.Join(conf.Warewulf.DataStore, "warewulf/bmc", ipmi.Template) + ipmi.Template = path.Join(conf.Paths.Datadir, "warewulf/bmc", ipmi.Template) } fbuf, err := os.ReadFile(ipmi.Template) if err != nil { diff --git a/internal/pkg/testenv/testenv.go b/internal/pkg/testenv/testenv.go index 2f3fce64..115688e9 100644 --- a/internal/pkg/testenv/testenv.go +++ b/internal/pkg/testenv/testenv.go @@ -30,7 +30,7 @@ type TestEnv struct { const Sysconfdir = "etc" const Bindir = "bin" -const Datadir = "share" +const Datadir = "usr/share" const Localstatedir = "var/local" const Srvdir = "srv" const Tftpdir = "srv/tftp" @@ -66,7 +66,7 @@ func New(t *testing.T) (env *TestEnv) { conf.Paths.Sysconfdir = env.GetPath(Sysconfdir) conf.Paths.Bindir = env.GetPath(Bindir) - conf.Warewulf.DataStore = env.GetPath(Datadir) + conf.Paths.Datadir = env.GetPath(Datadir) conf.Paths.Localstatedir = env.GetPath(Localstatedir) conf.Paths.Srvdir = env.GetPath(Srvdir) conf.TFTP.TftpRoot = env.GetPath(Tftpdir) @@ -81,7 +81,7 @@ func New(t *testing.T) (env *TestEnv) { for _, confPath := range []string{ conf.Paths.Sysconfdir, conf.Paths.Bindir, - conf.Warewulf.DataStore, + conf.Paths.Datadir, conf.Paths.Localstatedir, conf.Paths.Srvdir, conf.TFTP.TftpRoot, diff --git a/internal/pkg/upgrade/config.go b/internal/pkg/upgrade/config.go index ac597911..b14f7640 100644 --- a/internal/pkg/upgrade/config.go +++ b/internal/pkg/upgrade/config.go @@ -70,6 +70,14 @@ func (this *WarewulfYaml) Upgrade() (upgraded *config.WarewulfYaml) { if this.WWClient != nil { upgraded.WWClient = this.WWClient.Upgrade() } + if this.Warewulf != nil && this.Warewulf.DataStore != "" { + if upgraded.Paths == nil { + upgraded.Paths = new(config.BuildConfig) + } + if upgraded.Paths.Datadir == "" { + upgraded.Paths.Datadir = this.Warewulf.DataStore + } + } return upgraded } @@ -92,7 +100,6 @@ func (this *WarewulfConf) Upgrade() (upgraded *config.WarewulfConf) { upgraded.AutobuildOverlaysP = this.AutobuildOverlays upgraded.EnableHostOverlayP = this.EnableHostOverlay upgraded.SyslogP = this.Syslog - upgraded.DataStore = this.DataStore upgraded.GrubBootP = this.GrubBoot return upgraded } @@ -210,6 +217,7 @@ type BuildConfig struct { Srvdir string Firewallddir string Systemddir string + Datadir string WWOverlaydir string WWChrootdir string WWProvisiondir string @@ -225,6 +233,7 @@ func (this *BuildConfig) Upgrade() (upgraded *config.BuildConfig) { upgraded.Ipxesource = this.Ipxesource upgraded.Srvdir = this.Srvdir upgraded.Firewallddir = this.Firewallddir + upgraded.Datadir = this.Datadir upgraded.WWOverlaydir = this.WWOverlaydir upgraded.WWChrootdir = this.WWChrootdir upgraded.WWProvisiondir = this.WWProvisiondir diff --git a/internal/pkg/upgrade/config_test.go b/internal/pkg/upgrade/config_test.go index cd29511d..9bdd58f2 100644 --- a/internal/pkg/upgrade/config_test.go +++ b/internal/pkg/upgrade/config_test.go @@ -404,6 +404,106 @@ container mounts: - source: /etc/resolv.conf dest: /etc/resolv.conf readonly: true +`, + }, + { + name: "v4.6.0", + legacyYaml: ` +ipaddr: 10.0.0.1 +netmask: 255.255.252.0 +network: 10.0.0.0 +warewulf: + port: 9873 + secure: false + update interval: 60 + autobuild overlays: true + host overlay: true + syslog: false + datastore: /usr/share +dhcp: + enabled: true + range start: 10.0.1.1 + range end: 10.0.1.255 + systemd name: dhcpd +tftp: + enabled: true + systemd name: tftp + ipxe: + 00:0B: arm64-efi/snponly.efi + "00:00": undionly.kpxe + "00:07": ipxe-snponly-x86_64.efi + "00:09": ipxe-snponly-x86_64.efi +nfs: + enabled: true + export paths: + - path: /home + export options: rw,sync + mount options: defaults + mount: true + - path: /opt + export options: ro,sync,no_root_squash + mount options: defaults + mount: false + systemd name: nfs-server +ssh: + key types: + - rsa + - dsa + - ecdsa + - ed25519 +container mounts: + - source: /etc/resolv.conf + dest: /etc/resolv.conf + readonly: true +`, + upgradedYaml: ` +ipaddr: 10.0.0.1 +netmask: 255.255.252.0 +network: 10.0.0.0 +warewulf: + port: 9873 + secure: false + update interval: 60 + autobuild overlays: true + host overlay: true + syslog: false +dhcp: + enabled: true + range start: 10.0.1.1 + range end: 10.0.1.255 + systemd name: dhcpd +tftp: + enabled: true + systemd name: tftp + ipxe: + 00:0B: arm64-efi/snponly.efi + "00:00": undionly.kpxe + "00:07": ipxe-snponly-x86_64.efi + "00:09": ipxe-snponly-x86_64.efi +nfs: + enabled: true + export paths: + - path: /home + export options: rw,sync + mount options: defaults + mount: true + - path: /opt + export options: ro,sync,no_root_squash + mount options: defaults + mount: false + systemd name: nfs-server +ssh: + key types: + - rsa + - dsa + - ecdsa + - ed25519 +container mounts: + - source: /etc/resolv.conf + dest: /etc/resolv.conf + readonly: true +paths: + datadir: /usr/share `, }, } diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 7c4987b8..a40c1781 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -54,7 +54,8 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu build = util.PathIsNewer(stage_file, config.Get().Paths.NodesConf()) for _, overlayname := range stage_overlays { - build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) + overlayDir := overlay.GetOverlay(overlayname).Rootfs() + build = build || util.PathIsNewer(stage_file, overlayDir) } } diff --git a/overlays/debug/internal/debug_test.go b/overlays/debug/internal/debug_test.go index f7a90688..9c017dcf 100644 --- a/overlays/debug/internal/debug_test.go +++ b/overlays/debug/internal/debug_test.go @@ -177,7 +177,6 @@ data from other structures. - AutobuildOverlays: true - EnableHostOverlay: true - Syslog: false -- DataStore: REMOVED_BY_TEST ### Other nodes diff --git a/overlays/debug/rootfs/warewulf/template-variables.md.ww b/overlays/debug/rootfs/warewulf/template-variables.md.ww index 2f1d4b81..d863d633 100644 --- a/overlays/debug/rootfs/warewulf/template-variables.md.ww +++ b/overlays/debug/rootfs/warewulf/template-variables.md.ww @@ -117,7 +117,6 @@ data from other structures. - AutobuildOverlays: {{ .Warewulf.AutobuildOverlays }} - EnableHostOverlay: {{ .Warewulf.EnableHostOverlay }} - Syslog: {{ .Warewulf.Syslog }} -- DataStore: {{ .Warewulf.DataStore }} ### Other nodes diff --git a/userdocs/contents/overlays.rst b/userdocs/contents/overlays.rst index 7519c826..a71439be 100644 --- a/userdocs/contents/overlays.rst +++ b/userdocs/contents/overlays.rst @@ -2,34 +2,43 @@ Warewulf Overlays ================= -At this point we have discussed how Warewulf is designed to -scalably provision and manage thousands of cluster nodes by utilizing -identical stateless boot images. And there-in lies a problem to -solve. If these boot images are completely identical, then how do we -configure things like hostnames? IP addresses? Or any other node -specific custom configurations? +Warewulf is designed to scalably provision and manage thousands of cluster nodes by utilizing +identical stateless boot images. But if these boot images are completely identical, then how do we +configure things like hostnames? IP addresses? Or any other node-specific configurations? -While some of this can be managed by services like DHCP, and other -bits by configuration management, which can absolutely be done with -Warewulf and many people choose to do, these are heavy-weight -solutions to a simple problem to solve. +Some of configuration can be managed by services like DHCP. You can also use traditional +configuration management on a provisioned Warewulf cluster node. But these are heavy-weight +solutions to a simple problem. -Warewulf solves this with overlays and uses overlays in different ways -through the provisioning process. A node or profile can configure an -overlay in two different ways: +Warewulf addresses cluster node configuration with its **overlay** system. Overlays are collections +of files and templates that are rendered and built per-node and then applied over the container +image during the provisioning process. -* An overlay can be configured to run during boot as part of the - ``wwinit`` process. These overlays are called **system overlays**. -* An overlay can be configured to run periodically while the system is - running. These overlays are called **runtime overlays**. +System and runtime overlays +=========================== -The default profile has both a **wwinit** and a **runtime** overlay -configured. +A node or profile can configure an overlay in two different ways: -Overlays are compiled for each compute node individually. +* An overlay can be configured to apply only during boot as part of the ``wwinit`` process. These + overlays are called **system overlays**. +* An overlay can be configured to also apply periodically while the system is running. These overlays + are called **runtime overlays**. -Provided overlays -================= +Distribution and site overlays +============================== + +Warewulf also distinguishes between **distribution** overlays, which are included with Warewulf, and +**site** overlays, which are created or added locally. A site overlay always takes precedence over a +distribution overlay with the same name. Any modification of a distribution overlay with ``wwctl`` +actually makes changes to an automatically-generated **site** overlay cloned from the distribution +overlay. + +Site overlays are often stored at ``/var/lib/warewulf/overlays/``. Distribution overlays are often +stored at ``/usr/share/warewulf/overlays/``. But these paths are dependent on compilation, +distribution, packaging, and configuration settings. + +Provided distribution overlays +------------------------------ These overlays are provided as part of Warewulf. @@ -145,18 +154,14 @@ The following services get configuration files via the host overlay: ``exports.ww`` * the dhcpd service is configured with ``dhcpd.conf.ww`` -Combining Overlays -================== +Combining and overriding overlays +================================= -We recommended not changing the provided overlays. Rather, -add the changed files to a new overlay and combine them in the -configuration. This is possible as the configuration fields for the -system and runtime overlays are lists and can contain several -overlays. As an example for this, we will overwrite the -``/etc/issue`` file from the "issue" overlay. For this we will -create a new overlay called "welcome" and import the file ``/etc/issue`` -from the host to it. This overlay is then combined with the existing -overlays. +Multiple overlays can be applied to a single node, and overlays from multiple profiles are appended +together. The configuration fields for the system and runtime overlays are lists and can contain +several overlays. As an example for this, we will overwrite the ``/etc/issue`` file from the "issue" +overlay. For this we will create a new overlay called "welcome" and import the file ``/etc/issue`` +from the host to it. This overlay is then combined with the existing overlays. .. code-block:: console @@ -191,8 +196,8 @@ attributes. will be dropped, so ``/etc/hosts.ww`` will end up being ``/etc/hosts``. -Using Overlays -============== +Managing overlays +================= Warewulf includes a command group for manipulating overlays (``wwctl overlay``). With this you can add, edit, remove, change ownership, @@ -222,7 +227,7 @@ via the ``-o`` flag. On clusters with large numbers of nodes a significant speedup can be achieved by building overlays in parallel. Adding parallel overlay building to `wwctl` is planned, see issue `#1087 `_. -Until parallel overlay building is implemented, builds can be easily done in +Until parallel overlay building is implemented, builds can be easily done in parallel with Gnu `parallel` or `xargs`, for example: .. code-block:: console diff --git a/warewulf.spec.in b/warewulf.spec.in index 86dec4f7..ad943158 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -189,30 +189,30 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %dir %{_sharedstatedir}/warewulf %dir %{_sharedstatedir}/warewulf/chroots -%dir %{_sharedstatedir}/warewulf/overlays -%dir %{_sharedstatedir}/warewulf/overlays/* -%dir %{_sharedstatedir}/warewulf/overlays/*/rootfs -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/NetworkManager/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/debian.interfaces/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/debug/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/fstab/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/host/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/hostname/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/hosts/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ifcfg/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ignition/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/issue/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/netplan/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/resolv/rootfs/* -%attr(700, root, root) %{_sharedstatedir}/warewulf/overlays/ssh.authorized_keys/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ssh.host_keys/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/syncuser/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/systemd.netname/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/udev.netname/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wicked/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wwclient/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wwinit/rootfs/* -%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/localtime/rootfs/* +%dir %{_datadir}/warewulf/overlays +%dir %{_datadir}/warewulf/overlays/* +%dir %{_datadir}/warewulf/overlays/*/rootfs +%attr(-, root, root) %{_datadir}/warewulf/overlays/NetworkManager/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/debian.interfaces/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/debug/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/fstab/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/host/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/hostname/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/hosts/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/ifcfg/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/ignition/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/issue/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/netplan/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/resolv/rootfs/* +%attr(700, root, root) %{_datadir}/warewulf/overlays/ssh.authorized_keys/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/ssh.host_keys/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/syncuser/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/systemd.netname/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/udev.netname/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/wicked/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/wwclient/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/wwinit/rootfs/* +%attr(-, root, root) %{_datadir}/warewulf/overlays/localtime/rootfs/* %attr(-, root, root) %{_bindir}/wwctl %attr(-, root, root) %{_prefix}/lib/firewalld/services/warewulf.xml