Add methods to config.New().Paths for site and distribution overlays

- SiteOverlaySourcedir() uses WWOverlaydir (/var/lib/warewulf/overlays)
- DistributionOverlaySourcedir() uses Datadir (/usr/share/warewulf/overlays)
- config.New().Warewulf.DataStore moved to config.New().Paths.Datadir
- FindOverlays() ignores missing distribution overlay directory

Signed-off-by: Jonathon Anderson <janderson@ciq.com>

internal/app/wwctl/node/sensors/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/overlay/list/main_test.go

@@ -2,7 +2,6 @@ package list
 
 import (
 	"bytes"
-	"path"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,7 +13,7 @@ import (
 
 func Test_Overlay_List(t *testing.T) {
 	env := testenv.New(t)
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), `
+	env.WriteFile(t, "var/lib/warewulf/overlays/testoverlay/email.ww", `
 {{ if .Tags.email }}eMail: {{ .Tags.email }}{{else}} noMail{{- end }}
 `)
 	defer env.RemoveAll(t)

internal/app/wwctl/overlay/show/main_test.go

@@ -69,10 +69,10 @@ nodes:
       - empty
 `)
 
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/email.ww"), overlayEmail)
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "testoverlay/overlay.ww"), overlayOverlay)
-	env.WriteFile(t, path.Join(testenv.WWOverlaydir, "dist/foo.ww"), "foo")
-	env.WriteFile(t, path.Join(testenv.Sysconfdir, "overlays/dist/foo.ww"), "foobaar")
+	env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/email.ww", overlayEmail)
+	env.WriteFile(t, "usr/share/warewulf/overlays/testoverlay/overlay.ww", overlayOverlay)
+	env.WriteFile(t, "usr/share/warewulf/overlays/dist/foo.ww", "foo")
+	env.WriteFile(t, "var/lib/warewulf/overlays/dist/foo.ww", "foobaar")
 	defer env.RemoveAll(t)
 	warewulfd.SetNoDaemon()
 	t.Run("overlay show raw", func(t *testing.T) {

internal/app/wwctl/power/cycle/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/off/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/on/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/reset/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/soft/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/app/wwctl/power/status/root_test.go

@@ -28,7 +28,7 @@ nodes:
     - default
     ipmi:
       ipaddr: 10.10.10.10`)
-	env.ImportFile(t, "share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
+	env.ImportFile(t, "usr/share/warewulf/bmc/ipmitool.tmpl", "../../../../../lib/warewulf/bmc/ipmitool.tmpl")
 
 	tests := map[string]struct {
 		args     []string

internal/pkg/config/buildconfig.go.in

@@ -9,18 +9,19 @@ import (
 var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf"
 
 type BuildConfig struct {
-	Bindir         string `default:"@BINDIR@"`
-	Sysconfdir     string `default:"@SYSCONFDIR@"`
-	Localstatedir  string `default:"@LOCALSTATEDIR@"`
-	Cachedir       string `default:"@CACHEDIR@"`
-	Ipxesource     string `default:"@IPXESOURCE@"`
-	Srvdir         string `default:"@SRVDIR@"`
-	Firewallddir   string `default:"@FIREWALLDDIR@"`
-	Systemddir     string `default:"@SYSTEMDDIR@"`
-	WWOverlaydir   string `default:"@WWOVERLAYDIR@"`
-	WWChrootdir    string `default:"@WWCHROOTDIR@"`
-	WWProvisiondir string `default:"@WWPROVISIONDIR@"`
-	WWClientdir    string `default:"@WWCLIENTDIR@"`
+	Bindir         string `yaml:"bindir,omitempty" default:"@BINDIR@"`
+	Sysconfdir     string `yaml:"sysconfdir,omitempty" default:"@SYSCONFDIR@"`
+	Localstatedir  string `yaml:"localstatedir,omitempty" default:"@LOCALSTATEDIR@"`
+	Cachedir       string `yaml:"cachedir,omitempty" default:"@CACHEDIR@"`
+	Ipxesource     string `yaml:"ipxesource,omitempty" default:"@IPXESOURCE@"`
+	Srvdir         string `yaml:"srvdir,omitempty" default:"@SRVDIR@"`
+	Firewallddir   string `yaml:"firewallddir,omitempty" default:"@FIREWALLDDIR@"`
+	Systemddir     string `yaml:"systemddir,omitempty" default:"@SYSTEMDDIR@"`
+	Datadir        string `yaml:"datadir,omitempty" default:"@DATADIR@"`
+	WWOverlaydir   string `yaml:"wwoverlaydir,omitempty" default:"@WWOVERLAYDIR@"`
+	WWChrootdir    string `yaml:"wwchrootdir,omitempty" default:"@WWCHROOTDIR@"`
+	WWProvisiondir string `yaml:"wwprovisiondir,omitempty" default:"@WWPROVISIONDIR@"`
+	WWClientdir    string `yaml:"wwclientdir,omitempty" default:"@WWCLIENTDIR@"`
 }
 
 const Version = "@VERSION@"
@@ -41,14 +42,13 @@ func (this TFTPConf) Enabled() bool {
 // WarewulfConf adds additional Warewulf-specific configuration to
 // BaseConf.
 type WarewulfConf struct {
-	Port               int    `yaml:"port,omitempty" default:"9873"`
-	SecureP            *bool  `yaml:"secure,omitempty" default:"true"`
-	UpdateInterval     int    `yaml:"update interval,omitempty" default:"60"`
-	AutobuildOverlaysP *bool  `yaml:"autobuild overlays,omitempty" default:"true"`
-	EnableHostOverlayP *bool  `yaml:"host overlay,omitempty" default:"true"`
-	SyslogP            *bool  `yaml:"syslog,omitempty" default:"false"`
-	DataStore          string `yaml:"datastore,omitempty" default:"@DATADIR@"`
-	GrubBootP          *bool  `yaml:"grubboot,omitempty" default:"false"`
+	Port               int   `yaml:"port,omitempty" default:"9873"`
+	SecureP            *bool `yaml:"secure,omitempty" default:"true"`
+	UpdateInterval     int   `yaml:"update interval,omitempty" default:"60"`
+	AutobuildOverlaysP *bool `yaml:"autobuild overlays,omitempty" default:"true"`
+	EnableHostOverlayP *bool `yaml:"host overlay,omitempty" default:"true"`
+	SyslogP            *bool `yaml:"syslog,omitempty" default:"false"`
+	GrubBootP          *bool `yaml:"grubboot,omitempty" default:"false"`
 }
 
 func (this WarewulfConf) Secure() bool {
@@ -79,6 +79,14 @@ func (paths BuildConfig) OciBlobCachedir() string {
 	return path.Join(paths.Cachedir, "warewulf")
 }
 
+func (paths BuildConfig) SiteOverlaySourcedir() string {
+	return paths.WWOverlaydir
+}
+
+func (paths BuildConfig) DistributionOverlaySourcedir() string {
+	return path.Join(paths.Datadir, "warewulf", "overlays")
+}
+
 func (paths BuildConfig) OverlayProvisiondir() string {
 	return path.Join(paths.WWProvisiondir, "overlays")
 }

internal/pkg/oci/defaults.go

@@ -6,7 +6,7 @@ import (
 	warewulfconf "github.com/warewulf/warewulf/internal/pkg/config"
 )
 
-var defaultCachePath = filepath.Join(warewulfconf.Get().Warewulf.DataStore, "/container-cache/oci/")
+var defaultCachePath = filepath.Join(warewulfconf.Get().Paths.Datadir, "/container-cache/oci/")
 
 const (
 	blobPrefix   = "blobs"

internal/pkg/overlay/config.go

@@ -20,7 +20,7 @@ prefix in the overlay dir if this it exists
 func OverlaySourceDir(overlayName string) (overlaypath string, isSite bool) {
 	controller := warewulfconf.Get()
 	/* Assume using old style overlay dir without rootfs */
-	overlaypath = path.Join(controller.Paths.Sysconfdir, "overlays", overlayName)
+	overlaypath = path.Join(controller.Paths.SiteOverlaySourcedir(), overlayName)
 	if _, err := os.Stat(path.Join(overlaypath, "rootfs")); err == nil {
 		/* rootfs exists, use it. */
 		overlaypath = path.Join(overlaypath, "rootfs")
@@ -28,7 +28,7 @@ func OverlaySourceDir(overlayName string) (overlaypath string, isSite bool) {
 	if _, err := os.Stat(overlaypath); err == nil {
 		return overlaypath, true
 	}
-	overlaypath = path.Join(controller.Paths.WWOverlaydir, overlayName)
+	overlaypath = path.Join(controller.Paths.DistributionOverlaySourcedir(), overlayName)
 	if _, err := os.Stat(path.Join(overlaypath, "rootfs")); err == nil {
 		/* rootfs exists, use it. */
 		overlaypath = path.Join(overlaypath, "rootfs")
@@ -79,8 +79,8 @@ func (e *OverlayDoesNotExist) Error() string {
 // OverlayDoesNotExist error if distribution overlay doesn't exsist
 func CreateSiteOverlay(name string) (err error) {
 	controller := warewulfconf.Get()
-	distroPath := path.Join(controller.Paths.WWOverlaydir, name)
-	sitePath := path.Join(controller.Paths.Sysconfdir, "overlays", name)
+	distroPath := path.Join(controller.Paths.DistributionOverlaySourcedir(), name)
+	sitePath := path.Join(controller.Paths.SiteOverlaySourcedir(), name)
 	if !util.IsDir(distroPath) {
 		return &OverlayDoesNotExist{Name: name}
 	}

internal/pkg/overlay/overlay.go

@@ -88,12 +88,11 @@ Get all overlays present in warewulf
 func FindOverlays() (overlayList []string, err error) {
 	dotfilecheck, _ := regexp.Compile(`^\..*`)
 	controller := warewulfconf.Get()
-	files, err := os.ReadDir(controller.Paths.WWOverlaydir)
-	if err != nil {
-		return overlayList, fmt.Errorf("could not get list of distribution overlays: %w", err)
+	var files []fs.DirEntry
+	if distfiles, err := os.ReadDir(controller.Paths.DistributionOverlaySourcedir()); err == nil {
+		files = append(files, distfiles...)
 	}
-	sitefiles, err := os.ReadDir(path.Join(controller.Paths.Sysconfdir, "overlays"))
-	if err == nil { // we don't care if there are no site overlays
+	if sitefiles, err := os.ReadDir(path.Join(controller.Paths.SiteOverlaySourcedir())); err == nil {
 		files = append(files, sitefiles...)
 	}
 	for _, file := range files {

internal/pkg/power/ipmitool.go

@@ -37,7 +37,7 @@ func (ipmi *IPMI) getStr() (cmdStr string, err error) {
 	}
 	if !strings.HasPrefix(ipmi.Template, "/") {
 		conf := warewulfconf.Get()
-		ipmi.Template = path.Join(conf.Warewulf.DataStore, "warewulf/bmc", ipmi.Template)
+		ipmi.Template = path.Join(conf.Paths.Datadir, "warewulf/bmc", ipmi.Template)
 	}
 	fbuf, err := os.ReadFile(ipmi.Template)
 	if err != nil {

internal/pkg/testenv/testenv.go

@@ -30,7 +30,7 @@ type TestEnv struct {
 
 const Sysconfdir = "etc"
 const Bindir = "bin"
-const Datadir = "share"
+const Datadir = "usr/share"
 const Localstatedir = "var/local"
 const Srvdir = "srv"
 const Tftpdir = "srv/tftp"
@@ -66,7 +66,7 @@ func New(t *testing.T) (env *TestEnv) {
 
 	conf.Paths.Sysconfdir = env.GetPath(Sysconfdir)
 	conf.Paths.Bindir = env.GetPath(Bindir)
-	conf.Warewulf.DataStore = env.GetPath(Datadir)
+	conf.Paths.Datadir = env.GetPath(Datadir)
 	conf.Paths.Localstatedir = env.GetPath(Localstatedir)
 	conf.Paths.Srvdir = env.GetPath(Srvdir)
 	conf.TFTP.TftpRoot = env.GetPath(Tftpdir)
@@ -81,7 +81,7 @@ func New(t *testing.T) (env *TestEnv) {
 	for _, confPath := range []string{
 		conf.Paths.Sysconfdir,
 		conf.Paths.Bindir,
-		conf.Warewulf.DataStore,
+		conf.Paths.Datadir,
 		conf.Paths.Localstatedir,
 		conf.Paths.Srvdir,
 		conf.TFTP.TftpRoot,

internal/pkg/upgrade/config.go

@@ -70,6 +70,14 @@ func (this *WarewulfYaml) Upgrade() (upgraded *config.WarewulfYaml) {
 	if this.WWClient != nil {
 		upgraded.WWClient = this.WWClient.Upgrade()
 	}
+	if this.Warewulf != nil && this.Warewulf.DataStore != "" {
+		if upgraded.Paths == nil {
+			upgraded.Paths = new(config.BuildConfig)
+		}
+		if upgraded.Paths.Datadir == "" {
+			upgraded.Paths.Datadir = this.Warewulf.DataStore
+		}
+	}
 	return upgraded
 }
 
@@ -92,7 +100,6 @@ func (this *WarewulfConf) Upgrade() (upgraded *config.WarewulfConf) {
 	upgraded.AutobuildOverlaysP = this.AutobuildOverlays
 	upgraded.EnableHostOverlayP = this.EnableHostOverlay
 	upgraded.SyslogP = this.Syslog
-	upgraded.DataStore = this.DataStore
 	upgraded.GrubBootP = this.GrubBoot
 	return upgraded
 }
@@ -210,6 +217,7 @@ type BuildConfig struct {
 	Srvdir         string
 	Firewallddir   string
 	Systemddir     string
+	Datadir        string
 	WWOverlaydir   string
 	WWChrootdir    string
 	WWProvisiondir string
@@ -225,6 +233,7 @@ func (this *BuildConfig) Upgrade() (upgraded *config.BuildConfig) {
 	upgraded.Ipxesource = this.Ipxesource
 	upgraded.Srvdir = this.Srvdir
 	upgraded.Firewallddir = this.Firewallddir
+	upgraded.Datadir = this.Datadir
 	upgraded.WWOverlaydir = this.WWOverlaydir
 	upgraded.WWChrootdir = this.WWChrootdir
 	upgraded.WWProvisiondir = this.WWProvisiondir

internal/pkg/upgrade/config_test.go

@@ -404,6 +404,106 @@ container mounts:
   - source: /etc/resolv.conf
     dest: /etc/resolv.conf
     readonly: true
+`,
+	},
+	{
+		name: "v4.6.0",
+		legacyYaml: `
+ipaddr: 10.0.0.1
+netmask: 255.255.252.0
+network: 10.0.0.0
+warewulf:
+  port: 9873
+  secure: false
+  update interval: 60
+  autobuild overlays: true
+  host overlay: true
+  syslog: false
+  datastore: /usr/share
+dhcp:
+  enabled: true
+  range start: 10.0.1.1
+  range end: 10.0.1.255
+  systemd name: dhcpd
+tftp:
+  enabled: true
+  systemd name: tftp
+  ipxe:
+    00:0B: arm64-efi/snponly.efi
+    "00:00": undionly.kpxe
+    "00:07": ipxe-snponly-x86_64.efi
+    "00:09": ipxe-snponly-x86_64.efi
+nfs:
+  enabled: true
+  export paths:
+    - path: /home
+      export options: rw,sync
+      mount options: defaults
+      mount: true
+    - path: /opt
+      export options: ro,sync,no_root_squash
+      mount options: defaults
+      mount: false
+  systemd name: nfs-server
+ssh:
+  key types:
+    - rsa
+    - dsa
+    - ecdsa
+    - ed25519
+container mounts:
+  - source: /etc/resolv.conf
+    dest: /etc/resolv.conf
+    readonly: true
+`,
+		upgradedYaml: `
+ipaddr: 10.0.0.1
+netmask: 255.255.252.0
+network: 10.0.0.0
+warewulf:
+  port: 9873
+  secure: false
+  update interval: 60
+  autobuild overlays: true
+  host overlay: true
+  syslog: false
+dhcp:
+  enabled: true
+  range start: 10.0.1.1
+  range end: 10.0.1.255
+  systemd name: dhcpd
+tftp:
+  enabled: true
+  systemd name: tftp
+  ipxe:
+    00:0B: arm64-efi/snponly.efi
+    "00:00": undionly.kpxe
+    "00:07": ipxe-snponly-x86_64.efi
+    "00:09": ipxe-snponly-x86_64.efi
+nfs:
+  enabled: true
+  export paths:
+    - path: /home
+      export options: rw,sync
+      mount options: defaults
+      mount: true
+    - path: /opt
+      export options: ro,sync,no_root_squash
+      mount options: defaults
+      mount: false
+  systemd name: nfs-server
+ssh:
+  key types:
+    - rsa
+    - dsa
+    - ecdsa
+    - ed25519
+container mounts:
+  - source: /etc/resolv.conf
+    dest: /etc/resolv.conf
+    readonly: true
+paths:
+  datadir: /usr/share
 `,
 	},
 }

overlays/debug/internal/debug_test.go

@@ -177,7 +177,6 @@ data from other structures.
 - AutobuildOverlays: true
 - EnableHostOverlay: true
 - Syslog: false
-- DataStore: REMOVED_BY_TEST
 
 
 ### Other nodes

overlays/debug/rootfs/warewulf/template-variables.md.ww

@@ -117,7 +117,6 @@ data from other structures.
 - AutobuildOverlays: {{ .Warewulf.AutobuildOverlays }}
 - EnableHostOverlay: {{ .Warewulf.EnableHostOverlay }}
 - Syslog: {{ .Warewulf.Syslog }}
-- DataStore: {{ .Warewulf.DataStore }}
 
 
 ### Other nodes