Merge pull request #1558 from mslacken/SeperateUserOverlays

Introduce site overlays

CHANGELOG.md

@@ -81,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Provide detected kernel version to overlay templates. #1556
 - Bump github.com/containers/storage from 1.53.0 to 1.55.2 #1316, #892
 - Process nodes.conf path dynamically from config. #1595, #1596, #1569
+- Split overlays into distribution and site overlays. #831
 
 ### Removed
 

Makefile

@@ -96,9 +96,9 @@ install: build docs
 	install -d -m 0755 $(DESTDIR)$(BINDIR)
 	install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR)
 	install -d -m 0755 $(DESTDIR)$(WWPROVISIONDIR)
-	install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)
+	install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)
-	install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwclient/rootfs/$(WWCLIENTDIR)
+	install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwclient/rootfs/$(WWCLIENTDIR)
-	install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs/$(TFTPDIR)/warewulf/
+	install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs/$(TFTPDIR)/warewulf/
 	install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/examples
 	install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/ipxe
 	install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/grub
@@ -112,27 +112,27 @@ install: build docs
 	install -d -m 0755 $(DESTDIR)$(IPXESOURCE)
 	install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf
 	# wwctl genconfig to get the compiled in paths to warewulf.conf
-	install -d -m 0755 $(DESTDIR)$(WWDATADIR)/bmc
+	install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/bmc
 	test -f $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf || ./wwctl --warewulfconf etc/warewulf.conf genconfig warewulfconf print> $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf
 	test -f $(DESTDIR)$(WWCONFIGDIR)/nodes.conf || install -m 0644 etc/nodes.conf $(DESTDIR)$(WWCONFIGDIR)
 	for f in etc/examples/*.ww; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/examples/; done
 	for f in etc/ipxe/*.ipxe; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/ipxe/; done
-	for f in lib/warewulf/bmc/*.tmpl; do install -m 0644 $$f $(DESTDIR)$(WWDATADIR)/bmc; done
+	for f in lib/warewulf/bmc/*.tmpl; do install -m 0644 $$f $(DESTDIR)$(DATADIR)/warewulf/bmc; done
 	install -m 0644 etc/grub/grub.cfg.ww $(DESTDIR)$(WWCONFIGDIR)/grub/grub.cfg.ww
-	install -m 0644 etc/grub/chainload.ww $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs$(TFTPDIR)/warewulf/grub.cfg.ww
+	install -m 0644 etc/grub/chainload.ww $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs$(TFTPDIR)/warewulf/grub.cfg.ww
 	install -m 0644 etc/logrotate.d/warewulfd.conf $(DESTDIR)$(LOGROTATEDIR)/warewulfd.conf
-	(cd overlays && find * -path '*/internal' -prune -o -type f -exec install -D -m 0644 {} $(DESTDIR)$(WWOVERLAYDIR)/{} \;)
+	(cd overlays && find * -path '*/internal' -prune -o -type f -exec install -D -m 0644 {} $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;)
-	(cd overlays && find * -path '*/internal' -prune -o -type d -exec mkdir -pv $(DESTDIR)$(WWOVERLAYDIR)/{} \;)
+	(cd overlays && find * -path '*/internal' -prune -o -type d -exec mkdir -pv $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;)
-	(cd overlays && find * -path '*/internal' -prune -o -type l -exec cp -av {} $(DESTDIR)$(WWOVERLAYDIR)/{} \;)
+	(cd overlays && find * -path '*/internal' -prune -o -type l -exec cp -av {} $(DESTDIR)$(DATADIR)/warewulf/overlays/{} \;)
-	chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/init
+	chmod 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/init
-	chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/wwinit
+	chmod 0755 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/wwinit
-	chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/config.ww
+	chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/config.ww
-	chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/ssh.host_keys/rootfs/etc/ssh/ssh*
+	chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/ssh.host_keys/rootfs/etc/ssh/ssh*
-	chmod 0644 $(DESTDIR)$(WWOVERLAYDIR)/ssh.host_keys/rootfs/etc/ssh/ssh*.pub.ww
+	chmod 0644 $(DESTDIR)$(DATADIR)/warewulf/overlays/ssh.host_keys/rootfs/etc/ssh/ssh*.pub.ww
-	chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/NetworkManager/rootfs/etc/NetworkManager/system-connections/ww4-managed.ww
+	chmod 0600 $(DESTDIR)$(DATADIR)/warewulf/overlays/NetworkManager/rootfs/etc/NetworkManager/system-connections/ww4-managed.ww
-	chmod 0750 $(DESTDIR)$(WWOVERLAYDIR)/host/rootfs
+	chmod 0750 $(DESTDIR)$(DATADIR)/warewulf/overlays/host/rootfs
 	install -m 0755 wwctl $(DESTDIR)$(BINDIR)
-	install -m 0755 wwclient $(DESTDIR)$(WWOVERLAYDIR)/wwinit/rootfs/$(WWCLIENTDIR)/wwclient
+	install -m 0755 wwclient $(DESTDIR)$(DATADIR)/warewulf/overlays/wwinit/rootfs/$(WWCLIENTDIR)/wwclient
 	install -m 0644 include/firewalld/warewulf.xml $(DESTDIR)$(FIREWALLDDIR)
 	install -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR)
 	install -m 0644 LICENSE.md $(DESTDIR)$(WWDOCDIR)

Variables.mk

@@ -61,14 +61,13 @@ endif
 TFTPDIR ?= /var/lib/tftpboot
 
 # Warewulf directory paths
-VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR WWDATADIR IPXESOURCE
+VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR IPXESOURCE
 WWCONFIGDIR := $(SYSCONFDIR)/$(WAREWULF)
 WWPROVISIONDIR := $(LOCALSTATEDIR)/$(WAREWULF)/provision
 WWOVERLAYDIR := $(LOCALSTATEDIR)/$(WAREWULF)/overlays
 WWCHROOTDIR := $(LOCALSTATEDIR)/$(WAREWULF)/chroots
 WWTFTPDIR := $(TFTPDIR)/$(WAREWULF)
 WWDOCDIR := $(DOCDIR)/$(WAREWULF)
-WWDATADIR := $(DATADIR)/$(WAREWULF)
 WWCLIENTDIR ?= /warewulf
 
 CONFIG := $(shell pwd)

internal/app/wwctl/node/sensors/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/overlay/chmod/main.go

@@ -3,7 +3,6 @@ package chmod
 import (
 	"fmt"
 	"os"
-	"path"
 	"strconv"
 
 	"github.com/spf13/cobra"
@@ -12,8 +11,6 @@ import (
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
-	var overlaySourceDir string
-
 	overlayName := args[0]
 	fileName := args[1]
 
@@ -21,16 +18,19 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("could not convert requested mode: %s", err)
 	}
-
+	overlay_ := overlay.GetOverlay(overlayName)
-	overlaySourceDir = overlay.OverlaySourceDir(overlayName)
+	if !overlay_.IsSiteOverlay() {
-
+		overlay_, err = overlay_.CloneSiteOverlay()
-	if !util.IsDir(overlaySourceDir) {
+		if err != nil {
+			return err
+		}
+	}
+	if !overlay_.Exists() {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	overlayFile := path.Join(overlaySourceDir, fileName)
+	overlayFile := overlay_.File(fileName)
-
+	if !(util.IsFile(overlayFile) || util.IsDir(overlayFile)) {
-	if !util.IsFile(overlayFile) && !util.IsDir(overlayFile) {
 		return fmt.Errorf("file does not exist within overlay: %s:%s", overlayName, fileName)
 	}
 

internal/app/wwctl/overlay/chown/main.go

@@ -3,7 +3,6 @@ package chown
 import (
 	"fmt"
 	"os"
-	"path"
 	"strconv"
 
 	"github.com/warewulf/warewulf/internal/pkg/overlay"
@@ -13,7 +12,6 @@ import (
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
-	var overlaySourceDir string
 	var uid int
 	var gid int
 	var err error
@@ -35,15 +33,20 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		gid = -1
 	}
 
-	overlaySourceDir = overlay.OverlaySourceDir(overlayName)
+	overlay_ := overlay.GetOverlay(overlayName)
-
+	if !overlay_.Exists() {
-	if !util.IsDir(overlaySourceDir) {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	overlayFile := path.Join(overlaySourceDir, fileName)
+	if !overlay_.IsSiteOverlay() {
+		overlay_, err = overlay_.CloneSiteOverlay()
+		if err != nil {
+			return err
+		}
+	}
 
-	if !util.IsFile(overlayFile) && !util.IsDir(overlayFile) {
+	overlayFile := overlay_.File(fileName)
+	if !(util.IsFile(overlayFile) || util.IsDir(overlayFile)) {
 		return fmt.Errorf("file does not exist within overlay: %s:%s", overlayName, fileName)
 	}
 

internal/app/wwctl/overlay/create/main.go

@@ -6,6 +6,5 @@ import (
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) (err error) {
-	err = overlay.OverlayInit(args[0])
+	return overlay.GetSiteOverlay(args[0]).Create()
-	return
 }

internal/app/wwctl/overlay/delete/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"path"
 
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/warewulf/warewulf/internal/pkg/overlay"
 	"github.com/warewulf/warewulf/internal/pkg/util"
@@ -13,7 +12,6 @@ import (
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
-	var overlayPath string
 	var fileName string
 
 	overlayName := args[0]
@@ -22,27 +20,22 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		fileName = args[1]
 	}
 
-	overlayPath = overlay.OverlaySourceDir(overlayName)
+	overlay_ := overlay.GetOverlay(overlayName)
-
+	if overlay_.IsDistributionOverlay() {
-	if overlayPath == "" {
+		return fmt.Errorf("distribution overlay can't deleted")
-		return fmt.Errorf("overlay name did not resolve: '%s'", overlayName)
 	}
-
+	if !overlay_.Exists() {
-	if !util.IsDir(overlayPath) {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
 	if fileName == "" {
-		if overlayName == "wwinit" || overlayName == "host" {
-			return errors.New("refusing to delete the Warewulf overlay")
-		}
 		if Force {
-			err := os.RemoveAll(overlayPath)
+			err := os.RemoveAll(overlay_.Path())
 			if err != nil {
 				return fmt.Errorf("failed deleting overlay: %w", err)
 			}
 		} else {
-			err := os.Remove(overlayPath)
+			err := os.Remove(overlay_.Path())
 			if err != nil {
 				return fmt.Errorf("failed deleting overlay: %w", err)
 			}
@@ -50,28 +43,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		wwlog.Info("Deleted overlay: %s\n", args[0])
 
 	} else {
-		removePath := path.Join(overlayPath, fileName)
+		removePath := overlay_.File(fileName)
 
-		if !util.IsDir(removePath) && !util.IsFile(removePath) {
+		if !(util.IsDir(removePath) || util.IsFile(removePath)) {
 			return fmt.Errorf("path to remove doesn't exist in overlay: %s", removePath)
 		}
 
 		if Force {
 			err := os.RemoveAll(removePath)
 			if err != nil {
-				return fmt.Errorf("failed deleting file from overlay: %s:%s", overlayName, overlayPath)
+				return fmt.Errorf("failed deleting file from overlay: %s:%s", overlayName, removePath)
 			}
 		} else {
 			err := os.Remove(removePath)
 			if err != nil {
-				return fmt.Errorf("failed deleting overlay: %s:%s", overlayName, overlayPath)
+				return fmt.Errorf("failed deleting overlay: %s:%s", overlayName, removePath)
 			}
 		}
 
 		if Parents {
 			// Cleanup any empty directories left behind...
 			i := path.Dir(removePath)
-			for i != overlayPath {
+			for i != overlay_.Rootfs() {
 				wwlog.Debug("Evaluating directory to remove: %s", i)
 				err := os.Remove(i)
 				if err != nil {

internal/app/wwctl/overlay/edit/main.go

@@ -30,33 +30,22 @@ const initialTemplate = `# This is a Warewulf Template file.
 # Keep the following for better reference:
 # ---
 # This file is autogenerated by warewulf
-# Host:   {{.BuildHost}}
-# Time:   {{.BuildTime}}
-# Source: {{.BuildSource}}
 `
 
-func CobraRunE(cmd *cobra.Command, args []string) error {
+func CobraRunE(cmd *cobra.Command, args []string) (err error) {
 	overlayName := args[0]
 	fileName := args[1]
 
-	overlaySourceDir := overlay.OverlaySourceDir(overlayName)
+	overlay_ := overlay.GetOverlay(overlayName)
-	if !util.IsDir(overlaySourceDir) {
+	if !overlay_.Exists() {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	overlayFile := path.Join(overlaySourceDir, fileName)
+	overlayFile := overlay_.File(fileName)
 	wwlog.Debug("Will edit overlay file: %s", overlayFile)
-
 	overlayFileDir := path.Dir(overlayFile)
-	if CreateDirs {
+	if !(util.IsDir(overlayFileDir) || CreateDirs) {
-		err := os.MkdirAll(overlayFileDir, 0755)
+		return fmt.Errorf("%s does not exist. Use '--parents' option to create automatically", overlayFileDir)
-		if err != nil {
-			return fmt.Errorf("could not create directory: %s", overlayFileDir)
-		}
-	} else {
-		if !util.IsDir(overlayFileDir) {
-			return fmt.Errorf("%s does not exist. Use '--parents' option to create automatically", overlayFileDir)
-		}
 	}
 
 	tempFile, tempFileErr := os.CreateTemp("", "ww-overlay-edit-")
@@ -102,12 +91,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	} else {
 		if startTime == fileInfo.ModTime() {
 			wwlog.Debug("No change detected. Not updating overlay.")
-			os.Exit(0)
+			return nil
+		}
+	}
+
+	if !overlay_.IsSiteOverlay() {
+		overlay_, err = overlay_.CloneSiteOverlay()
+		if err != nil {
+			return err
+		}
+	}
+	// re-generate because overlay_ may have changed
+	overlayFile = overlay_.File(fileName)
+	overlayFileDir = path.Dir(overlayFile)
+
+	if CreateDirs {
+		if err := os.MkdirAll(overlayFileDir, 0755); err != nil {
+			return fmt.Errorf("could not create directory: %s", overlayFileDir)
 		}
 	}
 
 	// try renaming the tempfile to overlayfile first
-	err := os.Rename(tempFile.Name(), overlayFile)
+	err = os.Rename(tempFile.Name(), overlayFile)
 	if err != nil {
 		// if it fails, which probably means that they exists on different partitions
 		// fallback to data copy

internal/app/wwctl/overlay/imprt/main.go

@@ -13,9 +13,8 @@ import (
 	"github.com/warewulf/warewulf/internal/pkg/wwlog"
 )
 
-func CobraRunE(cmd *cobra.Command, args []string) error {
+func CobraRunE(cmd *cobra.Command, args []string) (err error) {
 	var dest string
-	var overlaySource string
 
 	overlayName := args[0]
 	source := args[1]
@@ -27,23 +26,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	}
 
 	wwlog.Verbose("Copying '%s' into overlay '%s:%s'", source, overlayName, dest)
-	overlaySource = overlay.OverlaySourceDir(overlayName)
+	overlay_ := overlay.GetOverlay(overlayName)
-
+	if !overlay_.IsSiteOverlay() {
-	if !util.IsDir(overlaySource) {
+		overlay_, err = overlay_.CloneSiteOverlay()
+		if err != nil {
+			return err
+		}
+	}
+	if !overlay_.Exists() {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	if util.IsDir(path.Join(overlaySource, dest)) {
+	if util.IsDir(overlay_.File(dest)) {
 		dest = path.Join(dest, path.Base(source))
 	}
 
-	if util.IsFile(path.Join(overlaySource, dest)) {
+	if util.IsFile(overlay_.File(dest)) {
 		return fmt.Errorf("a file with that name already exists in the overlay: %s", overlayName)
 	}
 
 	if CreateDirs {
-		parent := filepath.Dir(path.Join(overlaySource, dest))
+		parent := filepath.Dir(overlay_.File(dest))
-		if _, err := os.Stat(parent); os.IsNotExist(err) {
+		if _, err = os.Stat(parent); os.IsNotExist(err) {
 			wwlog.Debug("Create dir: %s", parent)
 			srcInfo, err := os.Stat(source)
 			if err != nil {
@@ -56,7 +60,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	err := util.CopyFile(source, path.Join(overlaySource, dest))
+	err = util.CopyFile(source, overlay_.File(dest))
 	if err != nil {
 		return fmt.Errorf("could not copy file into overlay: %w", err)
 	}

internal/app/wwctl/overlay/list/main.go

@@ -6,6 +6,7 @@ import (
 	"syscall"
 
 	"github.com/spf13/cobra"
+	"github.com/warewulf/warewulf/internal/app/wwctl/table"
 	"github.com/warewulf/warewulf/internal/pkg/overlay"
 	"github.com/warewulf/warewulf/internal/pkg/util"
 	"github.com/warewulf/warewulf/internal/pkg/wwlog"
@@ -24,51 +25,52 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	t := table.New(cmd.OutOrStdout())
 	if ListLong {
-		wwlog.Info("%-10s %5s %-5s %-18s %s\n", "PERM MODE", "UID", "GID", "SYSTEM-OVERLAY", "FILE PATH")
+		t.AddHeader("PERM MODE", "UID", "GID", "SYSTEM-OVERLAY", "FILE PATH", "SITE")
 	} else {
-		wwlog.Info("%-30s %-12s\n", "OVERLAY NAME", "FILES/DIRS")
+		t.AddHeader("OVERLAY NAME", "FILES/DIRS", "SITE")
 	}
 
-	for o := range overlays {
+	for _, name := range overlays {
-		name := overlays[o]
+		overlay_ := overlay.GetOverlay(name)
-		path := overlay.OverlaySourceDir(name)
 
-		if util.IsDir(path) {
+		if !overlay_.Exists() {
-			files := util.FindFiles(path)
+			wwlog.Error("system/%s (path not found:%s)", name, overlay_.Rootfs())
+			continue
+		}
 
-			wwlog.Debug("Iterating overlay path: %s", path)
+		files := util.FindFiles(overlay_.Rootfs())
-			if ListLong {
-				for file := range files {
-					s, err := os.Stat(files[file])
-					if err != nil {
-						continue
-					}
 
-					fileMode := s.Mode()
+		wwlog.Debug("Iterating overlay rootfs: %s", overlay_.Rootfs())
-					perms := fileMode & os.ModePerm
+		if ListLong {
-
+			for file := range files {
-					sys := s.Sys()
+				s, err := os.Stat(files[file])
-
+				if err != nil {
-					wwlog.Info("%v %5d %-5d %-18s /%s\n", perms, sys.(*syscall.Stat_t).Uid, sys.(*syscall.Stat_t).Gid, overlays[o], files[file])
+					continue
 				}
-			} else if ListContents {
+
-				var fileCount int
+				fileMode := s.Mode()
-				for file := range files {
+				perms := fileMode & os.ModePerm
-					wwlog.Info("%-30s /%-12s\n", name, files[file])
+
-					fileCount++
+				sys := s.Sys()
-				}
+
-				if fileCount == 0 {
+				t.AddLine(perms, sys.(*syscall.Stat_t).Uid, sys.(*syscall.Stat_t).Gid, name, files[file], overlay_.IsSiteOverlay())
-					wwlog.Info("%-30s %-12d\n", name, 0)
+			}
-				}
+		} else if ListContents {
-			} else {
+			var fileCount int
-				wwlog.Info("%-30s %-12d\n", name, len(files))
+			for file := range files {
+				t.AddLine(name, files[file], overlay_.IsSiteOverlay())
+				fileCount++
+			}
+			if fileCount == 0 {
+				t.AddLine(name, 0, overlay_.IsSiteOverlay())
 			}
-
 		} else {
-			wwlog.Error("system/%s (path not found:%s)", overlays[o], path)
+			t.AddLine(name, len(files), overlay_.IsSiteOverlay())
 		}
 	}
+	t.Print()
 
 	return nil
 }

internal/app/wwctl/overlay/list/main_test.go

@@ -2,7 +2,6 @@ package list
 
 import (
 	"bytes"
-	"path"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,7 +13,7 @@ import (
 
 func Test_Overlay_List(t *testing.T) {
 	env := testenv.New(t)
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), `
+	env.WriteFile(t, "var/lib/warewulf/overlays/testoverlay/email.ww", `
 {{ if .Tags.email }}eMail: {{ .Tags.email }}{{else}} noMail{{- end }}
 `)
 	defer env.RemoveAll(t)

internal/app/wwctl/overlay/mkdir/main.go

@@ -3,33 +3,33 @@ package mkdir
 import (
 	"fmt"
 	"os"
-	"path"
 
 	"github.com/spf13/cobra"
 	"github.com/warewulf/warewulf/internal/pkg/overlay"
-	"github.com/warewulf/warewulf/internal/pkg/util"
 	"github.com/warewulf/warewulf/internal/pkg/wwlog"
 )
 
-func CobraRunE(cmd *cobra.Command, args []string) error {
+func CobraRunE(cmd *cobra.Command, args []string) (err error) {
-	var overlaySourceDir string
-
 	overlayName := args[0]
 	dirName := args[1]
 
-	overlaySourceDir = overlay.OverlaySourceDir(overlayName)
+	overlay_ := overlay.GetOverlay(overlayName)
+	if !overlay_.IsSiteOverlay() {
+		overlay_, err = overlay_.CloneSiteOverlay()
+		if err != nil {
+			return err
+		}
+	}
 
-	if !util.IsDir(overlaySourceDir) {
+	if !overlay_.Exists() {
 		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	overlayDir := path.Join(overlaySourceDir, dirName)
+	overlayDir := overlay_.File(dirName)
-
 	wwlog.Debug("Will create directory in overlay: %s:%s", overlayName, dirName)
-
+	err = os.MkdirAll(overlayDir, os.FileMode(PermMode))
-	err := os.MkdirAll(overlayDir, os.FileMode(PermMode))
 	if err != nil {
-		return fmt.Errorf("could not create directory: %s", path.Dir(overlayDir))
+		return fmt.Errorf("could not create directory: %s", overlayDir)
 	}
 
 	return nil

internal/app/wwctl/overlay/show/main.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -18,18 +17,15 @@ import (
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
-	var overlaySourceDir string
-
 	overlayName := args[0]
 	fileName := args[1]
-	overlaySourceDir = overlay.OverlaySourceDir(overlayName)
 
-	if !util.IsDir(overlaySourceDir) {
+	overlay_ := overlay.GetOverlay(overlayName)
-		return fmt.Errorf("overlay dir: %s does not exist", overlaySourceDir)
+	if !overlay_.Exists() {
+		return fmt.Errorf("overlay does not exist: %s", overlayName)
 	}
 
-	overlayFile := path.Join(overlaySourceDir, fileName)
+	overlayFile := overlay_.File(fileName)
-
 	if !util.IsFile(overlayFile) {
 		return fmt.Errorf("file: %s does not exist within overlay", overlayFile)
 	}
@@ -61,7 +57,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 			nodeConf = node.NewNode(hostName)
 			nodeConf.ClusterName = hostName
 		}
-		tstruct, err := overlay.InitStruct(nodeConf)
+		tstruct, err := overlay.InitStruct(overlayName, nodeConf)
 		if err != nil {
 			return err
 		}

internal/app/wwctl/overlay/show/main_test.go

@@ -69,8 +69,10 @@ nodes:
       - empty
 `)
 
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), overlayEmail)
+	env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/email.ww", overlayEmail)
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/overlay.ww"), overlayOverlay)
+	env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/overlay.ww", overlayOverlay)
+	env.WriteFile(t, "usr/share/warewulf/overlays/dist/foo.ww", "foo")
+	env.WriteFile(t, "var/lib/warewulf/overlays/dist/foo.ww", "foobaar")
 	defer env.RemoveAll(t)
 	warewulfd.SetNoDaemon()
 	t.Run("overlay show raw", func(t *testing.T) {
@@ -128,6 +130,17 @@ nodes:
 		assert.NoError(t, err)
 		assert.Contains(t, buf.String(), "testoverlay")
 	})
+	t.Run("site overlays precede", func(t *testing.T) {
+		baseCmd.SetArgs([]string{"-r", "node1", "dist", "foo.ww"})
+		baseCmd := GetCommand()
+		buf := new(bytes.Buffer)
+		baseCmd.SetOut(buf)
+		baseCmd.SetErr(buf)
+		wwlog.SetLogWriter(buf)
+		err := baseCmd.Execute()
+		assert.NoError(t, err)
+		assert.Contains(t, buf.String(), "foobaar")
+	})
 }
 
 func TestShowServerTemplate(t *testing.T) {

internal/app/wwctl/power/cycle/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/off/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/on/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/reset/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/soft/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/status/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/pkg/config/buildconfig.go.in

@@ -9,18 +9,19 @@ import (
 var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf"
 
 type BuildConfig struct {
-	Bindir         string `default:"@BINDIR@"`
+	Bindir         string `yaml:"bindir,omitempty" default:"@BINDIR@"`
-	Sysconfdir     string `default:"@SYSCONFDIR@"`
+	Sysconfdir     string `yaml:"sysconfdir,omitempty" default:"@SYSCONFDIR@"`
-	Localstatedir  string `default:"@LOCALSTATEDIR@"`
+	Localstatedir  string `yaml:"localstatedir,omitempty" default:"@LOCALSTATEDIR@"`
-	Cachedir       string `default:"@CACHEDIR@"`
+	Cachedir       string `yaml:"cachedir,omitempty" default:"@CACHEDIR@"`
-	Ipxesource     string `default:"@IPXESOURCE@"`
+	Ipxesource     string `yaml:"ipxesource,omitempty" default:"@IPXESOURCE@"`
-	Srvdir         string `default:"@SRVDIR@"`
+	Srvdir         string `yaml:"srvdir,omitempty" default:"@SRVDIR@"`
-	Firewallddir   string `default:"@FIREWALLDDIR@"`
+	Firewallddir   string `yaml:"firewallddir,omitempty" default:"@FIREWALLDDIR@"`
-	Systemddir     string `default:"@SYSTEMDDIR@"`
+	Systemddir     string `yaml:"systemddir,omitempty" default:"@SYSTEMDDIR@"`
-	WWOverlaydir   string `default:"@WWOVERLAYDIR@"`
+	Datadir        string `yaml:"datadir,omitempty" default:"@DATADIR@"`
-	WWChrootdir    string `default:"@WWCHROOTDIR@"`
+	WWOverlaydir   string `yaml:"wwoverlaydir,omitempty" default:"@WWOVERLAYDIR@"`
-	WWProvisiondir string `default:"@WWPROVISIONDIR@"`
+	WWChrootdir    string `yaml:"wwchrootdir,omitempty" default:"@WWCHROOTDIR@"`
-	WWClientdir    string `default:"@WWCLIENTDIR@"`
+	WWProvisiondir string `yaml:"wwprovisiondir,omitempty" default:"@WWPROVISIONDIR@"`
+	WWClientdir    string `yaml:"wwclientdir,omitempty" default:"@WWCLIENTDIR@"`
 }
 
 const Version = "@VERSION@"
@@ -41,14 +42,13 @@ func (this TFTPConf) Enabled() bool {
 // WarewulfConf adds additional Warewulf-specific configuration to
 // BaseConf.
 type WarewulfConf struct {
-	Port               int    `yaml:"port,omitempty" default:"9873"`
+	Port               int   `yaml:"port,omitempty" default:"9873"`
-	SecureP            *bool  `yaml:"secure,omitempty" default:"true"`
+	SecureP            *bool `yaml:"secure,omitempty" default:"true"`
-	UpdateInterval     int    `yaml:"update interval,omitempty" default:"60"`
+	UpdateInterval     int   `yaml:"update interval,omitempty" default:"60"`
-	AutobuildOverlaysP *bool  `yaml:"autobuild overlays,omitempty" default:"true"`
+	AutobuildOverlaysP *bool `yaml:"autobuild overlays,omitempty" default:"true"`
-	EnableHostOverlayP *bool  `yaml:"host overlay,omitempty" default:"true"`
+	EnableHostOverlayP *bool `yaml:"host overlay,omitempty" default:"true"`
-	SyslogP            *bool  `yaml:"syslog,omitempty" default:"false"`
+	SyslogP            *bool `yaml:"syslog,omitempty" default:"false"`
-	DataStore          string `yaml:"datastore,omitempty" default:"@DATADIR@"`
+	GrubBootP          *bool `yaml:"grubboot,omitempty" default:"false"`
-	GrubBootP          *bool  `yaml:"grubboot,omitempty" default:"false"`
 }
 
 func (this WarewulfConf) Secure() bool {
@@ -79,6 +79,14 @@ func (paths BuildConfig) OciBlobCachedir() string {
 	return path.Join(paths.Cachedir, "warewulf")
 }
 
+func (paths BuildConfig) SiteOverlaydir() string {
+	return paths.WWOverlaydir
+}
+
+func (paths BuildConfig) DistributionOverlaydir() string {
+	return path.Join(paths.Datadir, "warewulf", "overlays")
+}
+
 func (paths BuildConfig) OverlayProvisiondir() string {
 	return path.Join(paths.WWProvisiondir, "overlays")
 }

internal/pkg/configure/hostfile.go

@@ -14,13 +14,14 @@ import (
 Creates '/etc/hosts' from the host template.
 */
 func Hostfile() (err error) {
-	hostTemplate := path.Join(overlay.OverlaySourceDir("host"), "/etc/hosts.ww")
+	overlay_ := overlay.GetOverlay("host")
+	hostTemplate := path.Join(overlay_.Rootfs(), "/etc/hosts.ww")
 	if !(util.IsFile(hostTemplate)) {
 		return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay")
 	}
 
 	hostname, _ := os.Hostname()
-	tstruct, err := overlay.InitStruct(node.NewNode(hostname))
+	tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname))
 	if err != nil {
 		return err
 	}

internal/pkg/oci/defaults.go

@@ -6,7 +6,7 @@ import (
 	warewulfconf "github.com/warewulf/warewulf/internal/pkg/config"
 )
 
-var defaultCachePath = filepath.Join(warewulfconf.Get().Warewulf.DataStore, "/container-cache/oci/")
+var defaultCachePath = filepath.Join(warewulfconf.Get().Paths.Datadir, "/container-cache/oci/")
 
 const (
 	blobPrefix   = "blobs"

internal/pkg/overlay/config.go

@@ -1,31 +1,74 @@
 package overlay
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"strings"
 
-	warewulfconf "github.com/warewulf/warewulf/internal/pkg/config"
+	"github.com/containers/storage/drivers/copy"
+
+	"github.com/warewulf/warewulf/internal/pkg/config"
+	"github.com/warewulf/warewulf/internal/pkg/util"
 	"github.com/warewulf/warewulf/internal/pkg/wwlog"
 )
 
-func OverlaySourceTopDir() string {
+// GetOverlay returns the filesystem path of an overlay identified by its name,
-	conf := warewulfconf.Get()
+// along with a boolean indicating whether the returned overlayPath corresponds
-	return conf.Paths.WWOverlaydir
+// to a site-specific overlay.
+func GetOverlay(name string) (overlay Overlay) {
+	overlay = GetSiteOverlay(name)
+	if overlay.Exists() {
+		return overlay
+	}
+	overlay = GetDistributionOverlay(name)
+	if overlay.Exists() {
+		return overlay
+	}
+	return GetSiteOverlay(name)
 }
 
-/*
+// GetDistributionOverlay returns the filesystem path of a distribution overlay
-Return the path for the base of the overlay, strips rootfs
+// identified by the given name.
-prefix in the overlay dir if this it exists
+func GetDistributionOverlay(name string) (overlay Overlay) {
-*/
+	return getOverlay(config.Get().Paths.DistributionOverlaydir(), name)
-func OverlaySourceDir(overlayName string) string {
+}
-	/* Assume using old style overlay dir without rootfs */
+
-	var overlaypath = path.Join(OverlaySourceTopDir(), overlayName)
+// GetSiteOverlay returns the filesystem path of a site-specific overlay
-	if _, err := os.Stat(path.Join(overlaypath, "rootfs")); err == nil {
+// identified by the given name.
-		/* rootfs exists, use it. */
+func GetSiteOverlay(name string) (overlay Overlay) {
-		overlaypath = path.Join(overlaypath, "rootfs")
+	return getOverlay(config.Get().Paths.SiteOverlaydir(), name)
+}
+
+// getOverlay constructs an overlay based on the given overlay directory and
+// overlay name. The overlay does not necessarily exist.
+func getOverlay(overlaydir, name string) (overlay Overlay) {
+	return Overlay(path.Join(overlaydir, name))
+}
+
+// Create creates a new overlay directory for the given overlay
+//
+// Returns an error if the overlay already exists or if directory creation fails.
+func (this Overlay) Create() error {
+	if util.IsDir(this.Path()) {
+		return fmt.Errorf("overlay already exists: %s", this)
 	}
-	return overlaypath
+	return os.MkdirAll(this.Rootfs(), 0755)
+}
+
+// Creates a site overlay from an existing distribution overlay.
+//
+// If the distribution overlay doesn't exist, return an error.
+func (this Overlay) CloneSiteOverlay() (siteOverlay Overlay, err error) {
+	siteOverlay = GetSiteOverlay(this.Name())
+	if !util.IsDir(this.Path()) {
+		return siteOverlay, fmt.Errorf("source overlay does not exist: %s", this.Name())
+	}
+	if siteOverlay.Exists() {
+		return siteOverlay, fmt.Errorf("site overlay already exists: %s", siteOverlay.Name())
+	}
+	err = copy.DirCopy(this.Path(), siteOverlay.Path(), copy.Content, true)
+	return siteOverlay, err
 }
 
 // OverlayImage returns the full path to an overlay image based on the
@@ -54,6 +97,5 @@ func OverlayImage(nodeName string, context string, overlayNames []string) string
 		return ""
 	}
 
-	conf := warewulfconf.Get()
+	return path.Join(config.Get().Paths.OverlayProvisiondir(), nodeName, name)
-	return path.Join(conf.Paths.OverlayProvisiondir(), nodeName, name)
 }

internal/pkg/overlay/datastructure.go

@@ -46,8 +46,9 @@ type TemplateStruct struct {
 /*
 Initialize an TemplateStruct with the given node.NodeInfo
 */
-func InitStruct(nodeData node.Node) (TemplateStruct, error) {
+func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) {
 	var tstruct TemplateStruct
+	tstruct.Overlay = overlayName
 	hostname, _ := os.Hostname()
 	tstruct.BuildHost = hostname
 	controller := warewulfconf.Get()

internal/pkg/overlay/overlay.go

@@ -14,8 +14,9 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
-
 	"github.com/pkg/errors"
+
+	"github.com/warewulf/warewulf/internal/pkg/config"
 	"github.com/warewulf/warewulf/internal/pkg/node"
 	"github.com/warewulf/warewulf/internal/pkg/util"
 	"github.com/warewulf/warewulf/internal/pkg/wwlog"
@@ -25,6 +26,82 @@ var (
 	ErrDoesNotExist = errors.New("overlay does not exist")
 )
 
+// Overlay represents an overlay directory path.
+type Overlay string
+
+// Name returns the base name of the overlay directory.
+//
+// This is derived from the full path of the overlay.
+func (this Overlay) Name() string {
+	return path.Base(this.Path())
+}
+
+// Path returns the string representation of the overlay path.
+//
+// This method allows the Overlay type to be easily converted back to its
+// underlying string representation.
+func (this Overlay) Path() string {
+	return string(this)
+}
+
+// Rootfs returns the path to the root filesystem (rootfs) within the overlay.
+//
+// If the "rootfs" directory exists inside the overlay path, it returns the
+// path to the "rootfs" directory. Otherwise, it checks if the overlay path
+// itself is a directory and returns that. If neither exists, it defaults to
+// returning the "rootfs" path.
+func (this Overlay) Rootfs() string {
+	rootfs := path.Join(this.Path(), "rootfs")
+	if util.IsDir(rootfs) {
+		return rootfs
+	} else if util.IsDir(this.Path()) {
+		return this.Path()
+	} else {
+		return rootfs
+	}
+}
+
+// File constructs a full path to a file within the overlay's root filesystem.
+//
+// Parameters:
+//   - filePath: The relative path of the file within the overlay.
+//
+// Returns:
+//   - The full path to the specified file in the overlay's rootfs.
+func (this Overlay) File(filePath string) string {
+	return path.Join(this.Rootfs(), filePath)
+}
+
+// Exists checks whether the overlay path exists and is a directory.
+//
+// Returns:
+//   - true if the overlay path exists and is a directory; false otherwise.
+func (this Overlay) Exists() bool {
+	return util.IsDir(this.Path())
+}
+
+// IsSiteOverlay determines whether the overlay is a site overlay.
+//
+// A site overlay is identified by its parent directory matching the configured
+// site overlay directory path.
+//
+// Returns:
+//   - true if the overlay is a site overlay; false otherwise.
+func (this Overlay) IsSiteOverlay() bool {
+	return path.Dir(this.Path()) == config.Get().Paths.SiteOverlaydir()
+}
+
+// IsDistributionOverlay determines whether the overlay is a distribution overlay.
+//
+// A distribution overlay is identified by its parent directory matching the configured
+// distribution overlay directory path.
+//
+// Returns:
+//   - true if the overlay is a distribution overlay; false otherwise.
+func (this Overlay) IsDistributionOverlay() bool {
+	return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir()
+}
+
 /*
 Build all overlays for a node
 */
@@ -70,7 +147,7 @@ func BuildHostOverlay() error {
 	hostname, _ := os.Hostname()
 	hostData := node.NewNode(hostname)
 	wwlog.Info("Building overlay for %s: host", hostname)
-	hostdir := OverlaySourceDir("host")
+	hostdir := GetOverlay("host").Rootfs()
 	stats, err := os.Stat(hostdir)
 	if err != nil {
 		return fmt.Errorf("could not build host overlay: %w ", err)
@@ -84,40 +161,26 @@ func BuildHostOverlay() error {
 /*
 Get all overlays present in warewulf
 */
-func FindOverlays() ([]string, error) {
+func FindOverlays() (overlayList []string, err error) {
-	var ret []string
 	dotfilecheck, _ := regexp.Compile(`^\..*`)
-
+	controller := config.Get()
-	files, err := os.ReadDir(OverlaySourceTopDir())
+	var files []fs.DirEntry
-	if err != nil {
+	if distfiles, err := os.ReadDir(controller.Paths.DistributionOverlaydir()); err == nil {
-		return ret, fmt.Errorf("could not get list of overlays: %w", err)
+		files = append(files, distfiles...)
+	}
+	if sitefiles, err := os.ReadDir(path.Join(controller.Paths.SiteOverlaydir())); err == nil {
+		files = append(files, sitefiles...)
 	}
-
 	for _, file := range files {
 		wwlog.Debug("Evaluating overlay source: %s", file.Name())
 		isdotfile := dotfilecheck.MatchString(file.Name())
 
 		if (file.IsDir()) && !(isdotfile) {
-			ret = append(ret, file.Name())
+			overlayList = append(overlayList, file.Name())
 		}
 	}
 
-	return ret, nil
+	return overlayList, nil
-}
-
-/*
-Creates an empty overlay
-*/
-func OverlayInit(overlayName string) error {
-	path := OverlaySourceDir(overlayName)
-
-	if util.IsDir(path) {
-		return errors.New("Overlay already exists: " + overlayName)
-	}
-
-	err := os.MkdirAll(path, 0755)
-
-	return err
 }
 
 /*
@@ -190,7 +253,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri
 	wwlog.Verbose("Processing node/overlay: %s/%s", nodeData.Id(), strings.Join(overlayNames, "-"))
 	for _, overlayName := range overlayNames {
 		wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir)
-		overlaySourceDir := OverlaySourceDir(overlayName)
+		overlaySourceDir := GetOverlay(overlayName).Rootfs()
 		wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir)
 		err := os.Chdir(overlaySourceDir)
 		if err != nil {
@@ -220,7 +283,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri
 				wwlog.Debug("Created directory in overlay: %s", location)
 
 			} else if filepath.Ext(location) == ".ww" {
-				tstruct, err := InitStruct(nodeData)
+				tstruct, err := InitStruct(overlayName, nodeData)
 				if err != nil {
 					return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err)
 				}
@@ -358,19 +421,6 @@ func RenderTemplateFile(fileName string, data TemplateStruct) (
 	err error) {
 	backupFile = true
 	writeFile = true
-	// parse the overlay name out of the absolute path
-	overlayPath, _ := filepath.Rel(OverlaySourceTopDir(), fileName)
-	withoutRootfs, err := filepath.Rel("rootfs", overlayPath)
-	if err != nil {
-		overlayPath = withoutRootfs
-	}
-	overlayPathElem := strings.Split(overlayPath, "/")
-	if len(overlayPathElem) > 1 {
-		data.Overlay = overlayPathElem[0]
-	} else {
-		data.Overlay = ""
-	}
-
 	// Build our FuncMap
 	funcMap := template.FuncMap{
 		"Include":      templateFileInclude,
@@ -442,7 +492,7 @@ func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
 
 // Get all the files as a string slice for a given overlay
 func OverlayGetFiles(name string) (files []string, err error) {
-	baseDir := OverlaySourceDir(name)
+	baseDir := GetOverlay(name).Rootfs()
 	if !util.IsDir(baseDir) {
 		err = fmt.Errorf("overlay %s doesn't exist", name)
 		return

internal/pkg/overlay/overlay_test.go

@@ -10,10 +10,106 @@ import (
 	"github.com/stretchr/testify/assert"
 	warewulfconf "github.com/warewulf/warewulf/internal/pkg/config"
 	"github.com/warewulf/warewulf/internal/pkg/node"
+	"github.com/warewulf/warewulf/internal/pkg/testenv"
 
 	"github.com/warewulf/warewulf/internal/pkg/util"
 )
 
+func Test_OverlayMethods(t *testing.T) {
+	env := testenv.New(t)
+	defer env.RemoveAll(t)
+
+	// Setup test data
+	sitedir := "/var/lib/warewulf/overlays/"
+	distdir := "/usr/share/warewulf/overlays/"
+
+	env.WriteFile(t, path.Join(sitedir, "siteonly/rootfs/testfile"), "a site overlay")
+	env.WriteFile(t, path.Join(distdir, "distonly/rootfs/testfile"), "a distribution overlay")
+	env.WriteFile(t, path.Join(sitedir, "legacy/testfile"), "a legacy overlay")
+	env.WriteFile(t, path.Join(sitedir, "both/rootfs/testfile"), "the site version")
+	env.WriteFile(t, path.Join(distdir, "both/rootfs/testfile"), "the distribution version")
+
+	var tests = map[string]struct {
+		name    string
+		path    string
+		rootfs  string
+		file    string
+		content string
+		exists  bool
+		isSite  bool
+		isDist  bool
+	}{
+		"site overlay": {
+			name:    "siteonly",
+			path:    path.Join(sitedir, "siteonly"),
+			rootfs:  path.Join(sitedir, "siteonly/rootfs"),
+			file:    path.Join(sitedir, "siteonly/rootfs/testfile"),
+			content: "a site overlay",
+			exists:  true,
+			isSite:  true,
+			isDist:  false,
+		},
+		"distribution overlay": {
+			name:    "distonly",
+			path:    path.Join(distdir, "distonly"),
+			rootfs:  path.Join(distdir, "distonly/rootfs"),
+			file:    path.Join(distdir, "distonly/rootfs/testfile"),
+			content: "a distribution overlay",
+			exists:  true,
+			isSite:  false,
+			isDist:  true,
+		},
+		"overlapping overlay": {
+			name:    "both",
+			path:    path.Join(sitedir, "both"),
+			rootfs:  path.Join(sitedir, "both/rootfs"),
+			file:    path.Join(sitedir, "both/rootfs/testfile"),
+			content: "the site version",
+			exists:  true,
+			isSite:  true,
+			isDist:  false,
+		},
+		"legacy overlay": {
+			name:    "legacy",
+			path:    path.Join(sitedir, "legacy"),
+			rootfs:  path.Join(sitedir, "legacy"),
+			file:    path.Join(sitedir, "legacy/testfile"),
+			content: "",
+			exists:  true,
+			isSite:  true,
+			isDist:  false,
+		},
+		"missing overlay": {
+			name:    "absent",
+			path:    path.Join(sitedir, "absent"),
+			rootfs:  path.Join(sitedir, "absent/rootfs"),
+			file:    path.Join(sitedir, "absent/rootfs/testfile"),
+			content: "",
+			exists:  false,
+			isSite:  true,
+			isDist:  false,
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			overlay := GetOverlay(tt.name)
+			assert.Equal(t, tt.name, overlay.Name())
+			assert.Equal(t, env.GetPath(tt.path), overlay.Path())
+			assert.Equal(t, env.GetPath(tt.rootfs), overlay.Rootfs())
+			assert.Equal(t, env.GetPath(tt.file), overlay.File("testfile"))
+			if tt.content != "" {
+				buffer, err := os.ReadFile(overlay.File("testfile"))
+				assert.NoError(t, err)
+				assert.Equal(t, tt.content, string(buffer))
+			}
+			assert.Equal(t, tt.exists, overlay.Exists())
+			assert.Equal(t, tt.isSite, overlay.IsSiteOverlay())
+			assert.Equal(t, tt.isDist, overlay.IsDistributionOverlay())
+		})
+	}
+}
+
 var buildOverlayTests = []struct {
 	description string
 	nodeName    string

internal/pkg/power/ipmitool.go

@@ -37,7 +37,7 @@ func (ipmi *IPMI) getStr() (cmdStr string, err error) {
 	}
 	if !strings.HasPrefix(ipmi.Template, "/") {
 		conf := warewulfconf.Get()
-		ipmi.Template = path.Join(conf.Warewulf.DataStore, "warewulf/bmc", ipmi.Template)
+		ipmi.Template = path.Join(conf.Paths.Datadir, "warewulf/bmc", ipmi.Template)
 	}
 	fbuf, err := os.ReadFile(ipmi.Template)
 	if err != nil {

internal/pkg/testenv/testenv.go

@@ -30,7 +30,7 @@ type TestEnv struct {
 
 const Sysconfdir = "etc"
 const Bindir = "bin"
-const Datadir = "share"
+const Datadir = "usr/share"
 const Localstatedir = "var/local"
 const Srvdir = "srv"
 const Tftpdir = "srv/tftp"
@@ -66,7 +66,7 @@ func New(t *testing.T) (env *TestEnv) {
 
 	conf.Paths.Sysconfdir = env.GetPath(Sysconfdir)
 	conf.Paths.Bindir = env.GetPath(Bindir)
-	conf.Warewulf.DataStore = env.GetPath(Datadir)
+	conf.Paths.Datadir = env.GetPath(Datadir)
 	conf.Paths.Localstatedir = env.GetPath(Localstatedir)
 	conf.Paths.Srvdir = env.GetPath(Srvdir)
 	conf.TFTP.TftpRoot = env.GetPath(Tftpdir)
@@ -81,7 +81,7 @@ func New(t *testing.T) (env *TestEnv) {
 	for _, confPath := range []string{
 		conf.Paths.Sysconfdir,
 		conf.Paths.Bindir,
-		conf.Warewulf.DataStore,
+		conf.Paths.Datadir,
 		conf.Paths.Localstatedir,
 		conf.Paths.Srvdir,
 		conf.TFTP.TftpRoot,

internal/pkg/upgrade/config.go

@@ -70,6 +70,14 @@ func (this *WarewulfYaml) Upgrade() (upgraded *config.WarewulfYaml) {
 	if this.WWClient != nil {
 		upgraded.WWClient = this.WWClient.Upgrade()
 	}
+	if this.Warewulf != nil && this.Warewulf.DataStore != "" {
+		if upgraded.Paths == nil {
+			upgraded.Paths = new(config.BuildConfig)
+		}
+		if upgraded.Paths.Datadir == "" {
+			upgraded.Paths.Datadir = this.Warewulf.DataStore
+		}
+	}
 	return upgraded
 }
 
@@ -92,7 +100,6 @@ func (this *WarewulfConf) Upgrade() (upgraded *config.WarewulfConf) {
 	upgraded.AutobuildOverlaysP = this.AutobuildOverlays
 	upgraded.EnableHostOverlayP = this.EnableHostOverlay
 	upgraded.SyslogP = this.Syslog
-	upgraded.DataStore = this.DataStore
 	upgraded.GrubBootP = this.GrubBoot
 	return upgraded
 }
@@ -210,6 +217,7 @@ type BuildConfig struct {
 	Srvdir         string
 	Firewallddir   string
 	Systemddir     string
+	Datadir        string
 	WWOverlaydir   string
 	WWChrootdir    string
 	WWProvisiondir string
@@ -225,6 +233,7 @@ func (this *BuildConfig) Upgrade() (upgraded *config.BuildConfig) {
 	upgraded.Ipxesource = this.Ipxesource
 	upgraded.Srvdir = this.Srvdir
 	upgraded.Firewallddir = this.Firewallddir
+	upgraded.Datadir = this.Datadir
 	upgraded.WWOverlaydir = this.WWOverlaydir
 	upgraded.WWChrootdir = this.WWChrootdir
 	upgraded.WWProvisiondir = this.WWProvisiondir

internal/pkg/upgrade/config_test.go

@@ -404,6 +404,106 @@ container mounts:
   - source: /etc/resolv.conf
     dest: /etc/resolv.conf
     readonly: true
+`,
+	},
+	{
+		name: "v4.6.0",
+		legacyYaml: `
+ipaddr: 10.0.0.1
+netmask: 255.255.252.0
+network: 10.0.0.0
+warewulf:
+  port: 9873
+  secure: false
+  update interval: 60
+  autobuild overlays: true
+  host overlay: true
+  syslog: false
+  datastore: /usr/share
+dhcp:
+  enabled: true
+  range start: 10.0.1.1
+  range end: 10.0.1.255
+  systemd name: dhcpd
+tftp:
+  enabled: true
+  systemd name: tftp
+  ipxe:
+    00:0B: arm64-efi/snponly.efi
+    "00:00": undionly.kpxe
+    "00:07": ipxe-snponly-x86_64.efi
+    "00:09": ipxe-snponly-x86_64.efi
+nfs:
+  enabled: true
+  export paths:
+    - path: /home
+      export options: rw,sync
+      mount options: defaults
+      mount: true
+    - path: /opt
+      export options: ro,sync,no_root_squash
+      mount options: defaults
+      mount: false
+  systemd name: nfs-server
+ssh:
+  key types:
+    - rsa
+    - dsa
+    - ecdsa
+    - ed25519
+container mounts:
+  - source: /etc/resolv.conf
+    dest: /etc/resolv.conf
+    readonly: true
+`,
+		upgradedYaml: `
+ipaddr: 10.0.0.1
+netmask: 255.255.252.0
+network: 10.0.0.0
+warewulf:
+  port: 9873
+  secure: false
+  update interval: 60
+  autobuild overlays: true
+  host overlay: true
+  syslog: false
+dhcp:
+  enabled: true
+  range start: 10.0.1.1
+  range end: 10.0.1.255
+  systemd name: dhcpd
+tftp:
+  enabled: true
+  systemd name: tftp
+  ipxe:
+    00:0B: arm64-efi/snponly.efi
+    "00:00": undionly.kpxe
+    "00:07": ipxe-snponly-x86_64.efi
+    "00:09": ipxe-snponly-x86_64.efi
+nfs:
+  enabled: true
+  export paths:
+    - path: /home
+      export options: rw,sync
+      mount options: defaults
+      mount: true
+    - path: /opt
+      export options: ro,sync,no_root_squash
+      mount options: defaults
+      mount: false
+  systemd name: nfs-server
+ssh:
+  key types:
+    - rsa
+    - dsa
+    - ecdsa
+    - ed25519
+container mounts:
+  - source: /etc/resolv.conf
+    dest: /etc/resolv.conf
+    readonly: true
+paths:
+  datadir: /usr/share
 `,
 	},
 }

internal/pkg/warewulfd/util.go

@@ -54,7 +54,8 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu
 		build = util.PathIsNewer(stage_file, config.Get().Paths.NodesConf())
 
 		for _, overlayname := range stage_overlays {
-			build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
+			overlayDir := overlay.GetOverlay(overlayname).Rootfs()
+			build = build || util.PathIsNewer(stage_file, overlayDir)
 		}
 	}
 

overlays/debug/internal/debug_test.go

@@ -177,7 +177,6 @@ data from other structures.
 - AutobuildOverlays: true
 - EnableHostOverlay: true
 - Syslog: false
-- DataStore: REMOVED_BY_TEST
 
 
 ### Other nodes

overlays/debug/rootfs/warewulf/template-variables.md.ww

@@ -117,7 +117,6 @@ data from other structures.
 - AutobuildOverlays: {{ .Warewulf.AutobuildOverlays }}
 - EnableHostOverlay: {{ .Warewulf.EnableHostOverlay }}
 - Syslog: {{ .Warewulf.Syslog }}
-- DataStore: {{ .Warewulf.DataStore }}
 
 
 ### Other nodes

userdocs/contents/overlays.rst

@@ -2,34 +2,43 @@
 Warewulf Overlays
 =================
 
-At this point we have discussed how Warewulf is designed to
+Warewulf is designed to scalably provision and manage thousands of cluster nodes by utilizing
-scalably provision and manage thousands of cluster nodes by utilizing
+identical stateless boot images. But if these boot images are completely identical, then how do we
-identical stateless boot images. And there-in lies a problem to
+configure things like hostnames? IP addresses? Or any other node-specific configurations?
-solve. If these boot images are completely identical, then how do we
-configure things like hostnames? IP addresses? Or any other node
-specific custom configurations?
 
-While some of this can be managed by services like DHCP, and other
+Some of configuration can be managed by services like DHCP. You can also use traditional
-bits by configuration management, which can absolutely be done with
+configuration management on a provisioned Warewulf cluster node. But these are heavy-weight
-Warewulf and many people choose to do, these are heavy-weight
+solutions to a simple problem.
-solutions to a simple problem to solve.
 
-Warewulf solves this with overlays and uses overlays in different ways
+Warewulf addresses cluster node configuration with its **overlay** system. Overlays are collections
-through the provisioning process. A node or profile can configure an
+of files and templates that are rendered and built per-node and then applied over the container
-overlay in two different ways:
+image during the provisioning process.
 
-* An overlay can be configured to run during boot as part of the
+System and runtime overlays
-  ``wwinit`` process. These overlays are called **system overlays**.
+===========================
-* An overlay can be configured to run periodically while the system is
-  running. These overlays are called **runtime overlays**.
 
-The default profile has both a **wwinit** and a **runtime** overlay
+A node or profile can configure an overlay in two different ways:
-configured.
 
-Overlays are compiled for each compute node individually.
+* An overlay can be configured to apply only during boot as part of the ``wwinit`` process. These
+  overlays are called **system overlays**.
+* An overlay can be configured to also apply periodically while the system is running. These overlays
+  are called **runtime overlays**.
 
-Provided overlays
+Distribution and site overlays
-=================
+==============================
+
+Warewulf also distinguishes between **distribution** overlays, which are included with Warewulf, and
+**site** overlays, which are created or added locally. A site overlay always takes precedence over a
+distribution overlay with the same name.  Any modification of a distribution overlay with ``wwctl``
+actually makes changes to an automatically-generated **site** overlay cloned from the distribution
+overlay.
+
+Site overlays are often stored at ``/var/lib/warewulf/overlays/``. Distribution overlays are often
+stored at ``/usr/share/warewulf/overlays/``. But these paths are dependent on compilation,
+distribution, packaging, and configuration settings.
+
+Provided distribution overlays
+------------------------------
 
 These overlays are provided as part of Warewulf.
 
@@ -145,18 +154,14 @@ The following services get configuration files via the host overlay:
   ``exports.ww``
 * the dhcpd service is configured with ``dhcpd.conf.ww``
 
-Combining Overlays
+Combining and overriding overlays
-==================
+=================================
 
-We recommended not changing the provided overlays. Rather,
+Multiple overlays can be applied to a single node, and overlays from multiple profiles are appended
-add the changed files to a new overlay and combine them in the
+together. The configuration fields for the system and runtime overlays are lists and can contain
-configuration. This is possible as the configuration fields for the
+several overlays. As an example for this, we will overwrite the ``/etc/issue`` file from the "issue"
-system and runtime overlays are lists and can contain several
+overlay. For this we will create a new overlay called "welcome" and import the file ``/etc/issue``
-overlays.  As an example for this, we will overwrite the
+from the host to it. This overlay is then combined with the existing overlays.
-``/etc/issue`` file from the "issue" overlay.  For this we will
-create a new overlay called "welcome" and import the file ``/etc/issue``
-from the host to it. This overlay is then combined with the existing
-overlays.
 
 .. code-block:: console
 
@@ -191,8 +196,8 @@ attributes.
    will be dropped, so ``/etc/hosts.ww`` will end up being
    ``/etc/hosts``.
 
-Using Overlays
+Managing overlays
-==============
+=================
 
 Warewulf includes a command group for manipulating overlays (``wwctl
 overlay``). With this you can add, edit, remove, change ownership,
@@ -222,7 +227,7 @@ via the ``-o`` flag.
 On clusters with large numbers of nodes a significant speedup can be achieved
 by building overlays in parallel. Adding parallel overlay building to `wwctl`
 is planned, see issue `#1087 <https://github.com/warewulf/warewulf/issues/1087>`_.
-Until parallel overlay building is implemented, builds can be easily done in 
+Until parallel overlay building is implemented, builds can be easily done in
 parallel with Gnu `parallel` or `xargs`, for example:
 
 .. code-block:: console

warewulf.spec.in

@@ -189,30 +189,30 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup}
 
 %dir %{_sharedstatedir}/warewulf
 %dir %{_sharedstatedir}/warewulf/chroots
-%dir %{_sharedstatedir}/warewulf/overlays
+%dir %{_datadir}/warewulf/overlays
-%dir %{_sharedstatedir}/warewulf/overlays/*
+%dir %{_datadir}/warewulf/overlays/*
-%dir %{_sharedstatedir}/warewulf/overlays/*/rootfs
+%dir %{_datadir}/warewulf/overlays/*/rootfs
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/NetworkManager/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/NetworkManager/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/debian.interfaces/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/debian.interfaces/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/debug/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/debug/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/fstab/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/fstab/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/host/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/host/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/hostname/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/hostname/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/hosts/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/hosts/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ifcfg/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/ifcfg/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ignition/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/ignition/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/issue/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/issue/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/netplan/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/netplan/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/resolv/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/resolv/rootfs/*
-%attr(700, root, root) %{_sharedstatedir}/warewulf/overlays/ssh.authorized_keys/rootfs/*
+%attr(700, root, root) %{_datadir}/warewulf/overlays/ssh.authorized_keys/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/ssh.host_keys/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/ssh.host_keys/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/syncuser/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/syncuser/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/systemd.netname/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/systemd.netname/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/udev.netname/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/udev.netname/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wicked/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/wicked/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wwclient/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/wwclient/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/wwinit/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/wwinit/rootfs/*
-%attr(-, root, root) %{_sharedstatedir}/warewulf/overlays/localtime/rootfs/*
+%attr(-, root, root) %{_datadir}/warewulf/overlays/localtime/rootfs/*
 
 %attr(-, root, root) %{_bindir}/wwctl
 %attr(-, root, root) %{_prefix}/lib/firewalld/services/warewulf.xml