diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fe6191..f088b583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Bump github.com/spf13/cobra from 1.7.0 to 1.8.0 #1166 - Bump github.com/fatih/color from 1.15.0 to 1.17.0 #1224 - Bump github.com/coreos/ignition/v2 from 2.15.0 to 2.19.0 #1239 -- Bump github.com/sassoftware/go-rpmutils from 0.2.0 to 0.4.0 #1217 - Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 #1481 - Bump google.golang.org/protobuf from 1.34.1 to 1.35.1 #1480 - Bump golang.org/x/term from 0.20.0 to 0.25.0 #1476 @@ -85,6 +84,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added note to booting userdoc for removing machine-id. #1609 - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 +- Parallelized overlay build. #1018 +- Parallelized and optimized overlay build. #1018 ### Removed diff --git a/go.mod b/go.mod index d30a0489..a4f400fd 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.0 require ( dario.cat/mergo v1.0.1 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/cavaliergopher/cpio v1.0.1 github.com/cheynewallace/tabby v1.1.1 github.com/containers/image/v5 v5.32.2 github.com/containers/storage v1.55.2 @@ -21,7 +22,6 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/umoci v0.4.7 github.com/pkg/errors v0.9.1 - github.com/sassoftware/go-rpmutils v0.4.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/talos-systems/go-smbios v0.1.1 diff --git a/go.sum b/go.sum index 8a26eaae..0f46fa29 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -310,8 +312,6 @@ github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZP github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg= -github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI= github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 2542f3c2..52c461b5 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -97,17 +97,17 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { return fmt.Errorf("could not open node configuration: %s", err) } - nodes, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } - nodes = node.FilterNodeListByName(nodes, []string{nodename}) - if len(nodes) != 1 { + filteredNodes := node.FilterNodeListByName(allNodes, []string{nodename}) + if len(filteredNodes) != 1 { return fmt.Errorf("no single node idendified with %s", nodename) } - overlays := nodes[0].SystemOverlay - overlays = append(overlays, nodes[0].RuntimeOverlay...) - err = overlay.BuildOverlayIndir(nodes[0], overlays, path.Join(runDir, "nodeoverlay")) + overlays := filteredNodes[0].SystemOverlay + overlays = append(overlays, filteredNodes[0].RuntimeOverlay...) + err = overlay.BuildOverlayIndir(filteredNodes[0], allNodes, overlays, path.Join(runDir, "nodeoverlay")) if err != nil { return fmt.Errorf("could not build overlay: %s", err) } diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 5fb89b62..2e987de7 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -18,18 +18,21 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("could not open node configuration: %s", err) } - db, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } + var filteredNodes []node.Node if len(args) > 0 { args = hostlist.Expand(args) - db = node.FilterNodeListByName(db, args) + filteredNodes = node.FilterNodeListByName(allNodes, args) - if len(db) < len(args) { + if len(filteredNodes) < len(args) { return errors.New("failed to find nodes") } + } else { + filteredNodes = allNodes } // NOTE: this is to keep backward compatible @@ -49,26 +52,25 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if len(args) > 0 { - if len(db) != 1 { + if len(filteredNodes) != 1 { return errors.New("must specify one node to build overlay") } - for _, node := range db { - return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) + for _, node := range filteredNodes { + return overlay.BuildOverlayIndir(node, allNodes, OverlayNames, OverlayDir) } } else { return errors.New("must specify a node to build overlay") } - } oldMask := syscall.Umask(007) defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames) + err = overlay.BuildSpecificOverlays(filteredNodes, allNodes, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db) + err = overlay.BuildAllOverlays(filteredNodes, allNodes, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index e24ad6b3..74055c6d 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -2,6 +2,7 @@ package build import ( "log" + "runtime" "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/node" @@ -31,6 +32,7 @@ var ( } OverlayNames []string OverlayDir string + Workers int ) func init() { @@ -44,7 +46,7 @@ func init() { } baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to the given directory. An overlay must also be ge given to use this option.`) - + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 8a890bb1..f53144f8 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}) + return overlay.BuildSpecificOverlays(updateNodes, nodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/imprt/root.go b/internal/app/wwctl/overlay/imprt/root.go index 6c7840d3..38359676 100644 --- a/internal/app/wwctl/overlay/imprt/root.go +++ b/internal/app/wwctl/overlay/imprt/root.go @@ -1,6 +1,8 @@ package imprt import ( + "runtime" + "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/overlay" ) @@ -24,11 +26,13 @@ var ( } NoOverlayUpdate bool CreateDirs bool + Workers int ) func init() { baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories") + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/show/main.go b/internal/app/wwctl/overlay/show/main.go index 97f7a3e4..533322e7 100644 --- a/internal/app/wwctl/overlay/show/main.go +++ b/internal/app/wwctl/overlay/show/main.go @@ -57,7 +57,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodeConf = node.NewNode(hostName) nodeConf.ClusterName = hostName } - tstruct, err := overlay.InitStruct(overlayName, nodeConf) + var allNodes []node.Node + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + tstruct, err := overlay.InitStruct(overlayName, nodeConf, allNodes) if err != nil { return err } diff --git a/internal/pkg/config/buildconfig.go.in b/internal/pkg/config/buildconfig.go.in index 45dd5969..9b3a5859 100644 --- a/internal/pkg/config/buildconfig.go.in +++ b/internal/pkg/config/buildconfig.go.in @@ -2,8 +2,6 @@ package config import ( "path" - - "github.com/warewulf/warewulf/internal/pkg/util" ) var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf" @@ -36,7 +34,7 @@ type TFTPConf struct { } func (this TFTPConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } // WarewulfConf adds additional Warewulf-specific configuration to @@ -52,23 +50,23 @@ type WarewulfConf struct { } func (this WarewulfConf) Secure() bool { - return util.BoolP(this.SecureP) + return BoolP(this.SecureP) } func (this WarewulfConf) AutobuildOverlays() bool { - return util.BoolP(this.AutobuildOverlaysP) + return BoolP(this.AutobuildOverlaysP) } func (this WarewulfConf) EnableHostOverlay() bool { - return util.BoolP(this.EnableHostOverlayP) + return BoolP(this.EnableHostOverlayP) } func (this WarewulfConf) Syslog() bool { - return util.BoolP(this.SyslogP) + return BoolP(this.SyslogP) } func (this WarewulfConf) GrubBoot() bool { - return util.BoolP(this.GrubBootP) + return BoolP(this.GrubBootP) } func (paths BuildConfig) NodesConf() string { diff --git a/internal/pkg/config/dhcp.go b/internal/pkg/config/dhcp.go index 56c70986..7149594d 100644 --- a/internal/pkg/config/dhcp.go +++ b/internal/pkg/config/dhcp.go @@ -1,9 +1,5 @@ package config -import ( - "github.com/warewulf/warewulf/internal/pkg/util" -) - // DHCPConf represents the configuration for the DHCP service that // Warewulf will configure. type DHCPConf struct { @@ -15,5 +11,5 @@ type DHCPConf struct { } func (this DHCPConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } diff --git a/internal/pkg/config/mounts.go b/internal/pkg/config/mounts.go index f228a4e5..bc4cf920 100644 --- a/internal/pkg/config/mounts.go +++ b/internal/pkg/config/mounts.go @@ -1,9 +1,5 @@ package config -import ( - "github.com/warewulf/warewulf/internal/pkg/util" -) - // A MountEntry represents a bind mount that is applied to a container // during exec and shell. type MountEntry struct { @@ -15,9 +11,9 @@ type MountEntry struct { } func (this MountEntry) ReadOnly() bool { - return util.BoolP(this.ReadOnlyP) + return BoolP(this.ReadOnlyP) } func (this MountEntry) Copy() bool { - return util.BoolP(this.CopyP) + return BoolP(this.CopyP) } diff --git a/internal/pkg/config/nfs.go b/internal/pkg/config/nfs.go index ca6fc8b3..061eb199 100644 --- a/internal/pkg/config/nfs.go +++ b/internal/pkg/config/nfs.go @@ -2,8 +2,6 @@ package config import ( "github.com/creasty/defaults" - - "github.com/warewulf/warewulf/internal/pkg/util" ) // NFSConf represents the NFS configuration that will be used by @@ -16,7 +14,7 @@ type NFSConf struct { } func (this NFSConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } // An NFSExportConf reprents a single NFS export / mount. @@ -28,7 +26,7 @@ type NFSExportConf struct { } func (this NFSExportConf) Mount() bool { - return util.BoolP(this.MountP) + return BoolP(this.MountP) } // Implements the Unmarshal interface for NFSConf to set default diff --git a/internal/pkg/config/util.go b/internal/pkg/config/util.go new file mode 100644 index 00000000..1198d46b --- /dev/null +++ b/internal/pkg/config/util.go @@ -0,0 +1,5 @@ +package config + +func BoolP(p *bool) bool { + return p != nil && *p +} diff --git a/internal/pkg/configure/hostfile.go b/internal/pkg/configure/hostfile.go index 3046eb41..e8bf1759 100644 --- a/internal/pkg/configure/hostfile.go +++ b/internal/pkg/configure/hostfile.go @@ -20,8 +20,18 @@ func Hostfile() (err error) { return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay") } + var allNodes []node.Node + if nodeDB, err := node.New(); err != nil { + return err + } else { + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + } + hostname, _ := os.Hostname() - tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname)) + tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname), allNodes) if err != nil { return err } diff --git a/internal/pkg/overlay/datastructure.go b/internal/pkg/overlay/datastructure.go index 67c61d5f..e818717f 100644 --- a/internal/pkg/overlay/datastructure.go +++ b/internal/pkg/overlay/datastructure.go @@ -46,16 +46,12 @@ type TemplateStruct struct { /* Initialize an TemplateStruct with the given node.NodeInfo */ -func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) { +func InitStruct(overlayName string, nodeData node.Node, allNodes []node.Node) (TemplateStruct, error) { var tstruct TemplateStruct tstruct.Overlay = overlayName hostname, _ := os.Hostname() tstruct.BuildHost = hostname controller := warewulfconf.Get() - nodeDB, err := node.New() - if err != nil { - return tstruct, err - } tstruct.ThisNode = &nodeData if tstruct.ThisNode.Kernel == nil { tstruct.ThisNode.Kernel = new(node.KernelConf) @@ -75,10 +71,6 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) tstruct.Ipaddr6 = controller.Ipaddr6 tstruct.Netmask = controller.Netmask tstruct.Network = controller.Network - allNodes, err := nodeDB.FindAllNodes() - if err != nil { - return tstruct, err - } // init some convenience vars tstruct.Id = nodeData.Id() tstruct.Hostname = nodeData.Id() @@ -92,14 +84,11 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) var buf bytes.Buffer enc := gob.NewEncoder(&buf) dec := gob.NewDecoder(&buf) - err = enc.Encode(nodeData) - if err != nil { + if err := enc.Encode(nodeData); err != nil { return tstruct, err } - err = dec.Decode(&tstruct) - if err != nil { + if err := dec.Decode(&tstruct); err != nil { return tstruct, err } return tstruct, nil - } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 5628d004..e0eba792 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -10,11 +10,10 @@ import ( "path/filepath" "regexp" "strings" - "syscall" + "sync" "text/template" "github.com/Masterminds/sprig/v3" - "github.com/pkg/errors" "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" @@ -23,7 +22,7 @@ import ( ) var ( - ErrDoesNotExist = errors.New("overlay does not exist") + ErrDoesNotExist = fmt.Errorf("overlay does not exist") ) // Overlay represents an overlay directory path. @@ -102,40 +101,76 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -/* -Build all overlays for a node -*/ -func BuildAllOverlays(nodes []node.Node) error { +func BuildAllOverlays(nodes []node.Node, allNodes []node.Node, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)*2) + + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) + if err := BuildOverlay(n, allNodes, "system", n.SystemOverlay); err != nil { + errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) + } + + wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) + if err := BuildOverlay(n, allNodes, "runtime", n.RuntimeOverlay); err != nil { + errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) + } + } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) - sysOverlays := n.SystemOverlay - wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(sysOverlays, ", ")) - err := BuildOverlay(n, "system", sysOverlays) - if err != nil { - return fmt.Errorf("could not build system overlays %v for node %s: %w", sysOverlays, n.Id(), err) - } - runOverlays := n.RuntimeOverlay - wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(runOverlays, ", ")) - err = BuildOverlay(n, "runtime", runOverlays) - if err != nil { - return fmt.Errorf("could not build runtime overlays %v for node %s: %w", runOverlays, n.Id(), err) - } + wg.Wait() + close(errChan) + for err := range errChan { + return err } return nil } -// TODO: Add an Overlay Delete for both sourcedir and image +func BuildSpecificOverlays(nodes []node.Node, allNodes []node.Node, overlayNames []string, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)) -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string) error { - for _, n := range nodes { - wwlog.Info("Building overlay for %s: %v", n, overlayNames) - for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) - if err != nil { - return fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building overlay for %s: %v", n, overlayNames) + for _, overlayName := range overlayNames { + err := BuildOverlay(n, allNodes, "", []string{overlayName}) + if err != nil { + errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + } } } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } + for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) + + wg.Wait() + close(errChan) + + for err := range errChan { + return err } return nil } @@ -155,7 +190,16 @@ func BuildHostOverlay() error { if !(stats.Mode() == os.FileMode(0750|os.ModeDir) || stats.Mode() == os.FileMode(0700|os.ModeDir)) { wwlog.SecWarn("Permissions of host overlay dir %s are %s (750 is considered as secure)", hostdir, stats.Mode()) } - return BuildOverlayIndir(hostData, []string{"host"}, "/") + registry, err := node.New() + if err != nil { + return err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + return err + } + return BuildOverlayIndir(hostData, allNodes, []string{"host"}, "/") } /* @@ -179,14 +223,13 @@ func FindOverlays() (overlayList []string, err error) { overlayList = append(overlayList, file.Name()) } } - return overlayList, nil } /* Build the given overlays for a node and create a Image for them */ -func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) error { +func BuildOverlay(nodeConf node.Node, allNodes []node.Node, context string, overlayNames []string) error { if len(overlayNames) == 0 && context == "" { return nil } @@ -211,7 +254,7 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err wwlog.Debug("Created temporary directory for %s: %s", name, buildDir) - err = BuildOverlayIndir(nodeConf, overlayNames, buildDir) + err = BuildOverlayIndir(nodeConf, allNodes, overlayNames, buildDir) if err != nil { return fmt.Errorf("failed to generate files for %s: %w", name, err) } @@ -231,155 +274,157 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err return err } -/* -Build the given overlays for a node in the given directory. If the given does not -exists it will be created. -*/ -func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error { +var regFile *regexp.Regexp +var regLink *regexp.Regexp + +func init() { + regFile = regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`) + regLink = regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`) +} + +// Build the given overlays for a node in the given directory. +func BuildOverlayIndir(nodeData node.Node, allNodes []node.Node, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return nil } if !util.IsDir(outputDir) { - return errors.Errorf("output must a be a directory: %s", outputDir) + return fmt.Errorf("output must a be a directory: %s", outputDir) } if !util.ValidString(strings.Join(overlayNames, ""), "^[a-zA-Z0-9-._:]+$") { - return errors.Errorf("overlay names contains illegal characters: %v", overlayNames) + return fmt.Errorf("overlay names contains illegal characters: %v", overlayNames) } - // Temporarily set umask to 0000, so directories in the overlay retain permissions - defer syscall.Umask(syscall.Umask(0)) - - wwlog.Verbose("Processing node/overlay: %s/%s", nodeData.Id(), strings.Join(overlayNames, "-")) + wwlog.Verbose("Processing node/overlays: %s/%s", nodeData.Id(), strings.Join(overlayNames, ",")) for _, overlayName := range overlayNames { wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir) - overlaySourceDir := GetOverlay(overlayName).Rootfs() - wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir) - err := os.Chdir(overlaySourceDir) - if err != nil { - return fmt.Errorf("directory: %s name: %s err: %w", overlaySourceDir, overlayName, ErrDoesNotExist) + overlayRootfs := GetOverlay(overlayName).Rootfs() + if !util.IsDir(overlayRootfs) { + return fmt.Errorf("overlay %s: %w", overlayName, ErrDoesNotExist) } - wwlog.Verbose("Walking the overlay structure: %s", overlaySourceDir) - err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + wwlog.Debug("Walking the overlay structure: %s", overlayRootfs) + err := filepath.Walk(overlayRootfs, func(walkPath string, info os.FileInfo, err error) error { if err != nil { - return fmt.Errorf("error for %s: %w", location, err) + return fmt.Errorf("error for %s: %w", walkPath, err) } + wwlog.Debug("Found overlay file: %s", walkPath) - wwlog.Debug("Found overlay file: %s", location) + relPath, relErr := filepath.Rel(overlayRootfs, walkPath) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", walkPath, relErr) + return relErr + } + outputPath := path.Join(outputDir, relPath) if info.IsDir() { - wwlog.Debug("Found directory: %s", location) + wwlog.Debug("Found directory: %s", walkPath) - err = os.MkdirAll(path.Join(outputDir, location), info.Mode()) - if err != nil { + if err = os.MkdirAll(outputPath, info.Mode()); err != nil { return fmt.Errorf("could not create directory within overlay: %w", err) } - err = util.CopyUIDGID(location, path.Join(outputDir, location)) - if err != nil { + if err = util.CopyUIDGID(walkPath, outputPath); err != nil { return fmt.Errorf("failed setting permissions on overlay directory: %w", err) } - wwlog.Debug("Created directory in overlay: %s", location) + wwlog.Debug("Created directory in overlay: %s", outputPath) - } else if filepath.Ext(location) == ".ww" { - tstruct, err := InitStruct(overlayName, nodeData) + } else if filepath.Ext(walkPath) == ".ww" { + originalOutputPath := outputPath + outputPath := strings.TrimSuffix(outputPath, ".ww") + tstruct, err := InitStruct(overlayName, nodeData, allNodes) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } - tstruct.BuildSource = path.Join(overlaySourceDir, location) - wwlog.Verbose("Evaluating overlay template file: %s", location) - destFile := strings.TrimSuffix(location, ".ww") + tstruct.BuildSource = walkPath + wwlog.Verbose("Evaluating overlay template file: %s", walkPath) - buffer, backupFile, writeFile, err := RenderTemplateFile(location, tstruct) + buffer, backupFile, writeFile, err := RenderTemplateFile(walkPath, tstruct) if err != nil { - return fmt.Errorf("failed to render template %s: %w", location, err) + return fmt.Errorf("failed to render template %s: %w", walkPath, err) } - if writeFile { - destFileName := destFile - var fileBuffer bytes.Buffer - // search for magic file name comment - fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) - fileScanner.Split(ScanLines) - regFile := regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`) - regLink := regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`) - foundFileComment := false - for fileScanner.Scan() { - line := fileScanner.Text() - filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1) - softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1) - if len(softlinkFromTemplate) != 0 { - wwlog.Debug("Creating soft link %s -> %s", destFileName, softlinkFromTemplate[0][1]) - return os.Symlink(softlinkFromTemplate[0][1], path.Join(outputDir, destFileName)) - } else if len(filenameFromTemplate) != 0 { - wwlog.Debug("Writing file %s", filenameFromTemplate[0][1]) - if foundFileComment { - err = CarefulWriteBuffer(path.Join(outputDir, destFileName), - fileBuffer, backupFile, info.Mode()) - if err != nil { - return fmt.Errorf("could not write file from template: %w", err) - } - err = util.CopyUIDGID(location, path.Join(outputDir, destFileName)) - if err != nil { - return fmt.Errorf("failed setting permissions on template output file: %w", err) - } - fileBuffer.Reset() + if !writeFile { + return nil + } + var fileBuffer bytes.Buffer + // search for magic file name comment + fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) + fileScanner.Split(ScanLines) + foundFileComment := false + for fileScanner.Scan() { + line := fileScanner.Text() + filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1) + softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1) + if len(softlinkFromTemplate) != 0 { + wwlog.Debug("Creating soft link %s -> %s", outputPath, softlinkFromTemplate[0][1]) + return os.Symlink(softlinkFromTemplate[0][1], outputPath) + } else if len(filenameFromTemplate) != 0 { + wwlog.Debug("Writing file %s", filenameFromTemplate[0][1]) + if foundFileComment { + err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode()) + if err != nil { + return fmt.Errorf("could not write file from template: %w", err) } - destFileName = path.Join(path.Dir(destFile), filenameFromTemplate[0][1]) - foundFileComment = true - } else { - _, _ = fileBuffer.WriteString(line) + err = util.CopyUIDGID(walkPath, outputPath) + if err != nil { + return fmt.Errorf("failed setting permissions on template output file: %w", err) + } + fileBuffer.Reset() + } + outputPath = path.Join(path.Dir(originalOutputPath), filenameFromTemplate[0][1]) + foundFileComment = true + } else { + if _, err = fileBuffer.WriteString(line); err != nil { + return fmt.Errorf("could not write to template buffer: %w", err) } } - err = CarefulWriteBuffer(path.Join(outputDir, destFileName), fileBuffer, backupFile, info.Mode()) - if err != nil { - return fmt.Errorf("could not write file from template: %w", err) - } - err = util.CopyUIDGID(location, path.Join(outputDir, destFileName)) - if err != nil { - return fmt.Errorf("failed setting permissions on template output file: %w", err) - } - - wwlog.Debug("Wrote template file into overlay: %s", destFile) - - // } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b { - // wwlog.Debug("Ignoring WW template file: %s", location) } + err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode()) + if err != nil { + return fmt.Errorf("could not write file from template: %w", err) + } + err = util.CopyUIDGID(walkPath, outputPath) + if err != nil { + return fmt.Errorf("failed setting permissions on template output file: %w", err) + } + wwlog.Debug("Wrote template file into overlay: %s", outputPath) + } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { - wwlog.Debug("Found symlink %s", location) - destination, err := os.Readlink(location) + wwlog.Debug("Found symlink %s", walkPath) + target, err := os.Readlink(walkPath) if err != nil { - wwlog.ErrorExc(err, "") + return fmt.Errorf("failed reading symlink: %w", err) } - if util.IsFile(path.Join(outputDir, location)) { - if !util.IsFile(path.Join(outputDir, location+".wwbackup")) { - wwlog.Debug("Target exists, creating backup file") - err = os.Rename(path.Join(outputDir, location), path.Join(outputDir, location+".wwbackup")) + if util.IsFile(outputPath) { + backupPath := outputPath + ".wwbackup" + if !util.IsFile(backupPath) { + wwlog.Debug("Output file already exists: moving to backup file") + if err = os.Rename(outputPath, backupPath); err != nil { + return fmt.Errorf("failed renaming to backup file: %w", err) + } } else { - wwlog.Debug("%s exists, keeping the backup file", path.Join(outputDir, location+".wwbackup")) - err = os.Remove(path.Join(outputDir, location)) - } - if err != nil { - wwlog.ErrorExc(err, "") + wwlog.Debug("%s exists, keeping the backup file", backupPath) + if err = os.Remove(outputPath); err != nil { + return fmt.Errorf("failed removing existing file: %w", err) + } } } - err = os.Symlink(destination, path.Join(outputDir, location)) - if err != nil { - wwlog.ErrorExc(err, "") + if err = os.Symlink(target, outputPath); err != nil { + return fmt.Errorf("failed creating symlink: %w", err) } + wwlog.Debug("Created symlink file: %s", outputPath) } else { - err := util.CopyFile(location, path.Join(outputDir, location)) - if err == nil { - wwlog.Debug("Copied file into overlay: %s", location) - } else { + if err := util.CopyFile(walkPath, outputPath); err != nil { return fmt.Errorf("could not copy file into overlay: %w", err) } + wwlog.Debug("Copied overlay file: %s", outputPath) } return nil }) + if err != nil { - return fmt.Errorf("failed to build overlay working directory: %w", err) + return fmt.Errorf("failed to build overlay image directory: %w", err) } } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index a409041d..af6a09af 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -1,9 +1,11 @@ package overlay import ( + "fmt" "io" "os" "path" + "runtime" "sort" "testing" @@ -11,8 +13,9 @@ import ( warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/testenv" - "github.com/warewulf/warewulf/internal/pkg/util" + + "github.com/cavaliergopher/cpio" ) func Test_OverlayMethods(t *testing.T) { @@ -110,158 +113,310 @@ func Test_OverlayMethods(t *testing.T) { } } -var buildOverlayTests = []struct { - description string - nodeName string - context string - overlays []string - image string - contents []string - hasFiles bool -}{ - { - description: "if no node, context, or overlays are specified then no overlay image is generated", - nodeName: "", - context: "", - overlays: nil, - image: "", - contents: nil, - }, - { - description: "if only node is specified then no overlay image is generated", - nodeName: "node1", - context: "", - overlays: nil, - image: "", - contents: nil, - }, - { - description: "if only context is specified then context named image is generated", - nodeName: "", - context: "system", - overlays: nil, - image: "__SYSTEM__.img", - contents: nil, - }, - { - description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory", - nodeName: "", - context: "", - overlays: []string{"o1"}, - image: "o1.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory", - nodeName: "", - context: "", - overlays: []string{"o1", "o2"}, - image: "o1-o2.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if a single node overlay is specified, then the overlay is built in a node overlay directory", - nodeName: "node1", - context: "", - overlays: []string{"o1"}, - image: "node1/o1.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory", - nodeName: "node1", - context: "", - overlays: []string{"o1", "o2"}, - image: "node1/o1-o2.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if no node system overlays are specified, then context pointed overlay is generated", - nodeName: "node1", - context: "system", - overlays: nil, - image: "node1/__SYSTEM__.img", - contents: nil, - }, - { - description: "if no node runtime overlays are specified, then context pointed overlay is generated", - nodeName: "node1", - context: "runtime", - overlays: nil, - image: "node1/__RUNTIME__.img", - contents: nil, - }, - { - description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory", - nodeName: "node1", - context: "system", - overlays: []string{"o1"}, - image: "node1/__SYSTEM__.img", - contents: []string{"o1.txt"}, - }, - { - description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory", - nodeName: "node1", - context: "runtime", - overlays: []string{"o1"}, - image: "node1/__RUNTIME__.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays", - nodeName: "node1", - context: "system", - overlays: []string{"o1", "o2"}, - image: "node1/__SYSTEM__.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays", - nodeName: "node1", - context: "runtime", - overlays: []string{"o1", "o2"}, - image: "node1/__RUNTIME__.img", - contents: []string{"o1.txt", "o2.txt"}, - }, +func Test_BuildOverlayIndir(t *testing.T) { + var tests = map[string]struct { + node node.Node + overlays []string + overlayFiles map[string]string + overlayDirs []string + overlaySymlinks map[string]string + outputDir string + outputFiles map[string]string + outputDirs []string + outputSymlinks map[string]string + }{ + "empty": { + outputDir: "/image", + }, + "empty directory": { + overlays: []string{"o1"}, + overlayDirs: []string{"/var/lib/warewulf/overlays/o1/rootfs/testdir"}, + outputDir: "/image", + outputDirs: []string{"testdir"}, + }, + "single flat file": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/testfile": "A test file from o1", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "testfile": "A test file from o1", + }, + }, + "multiple overlays": { + overlays: []string{"o1", "o2"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/t1.txt": "A test file from o1", + "/var/lib/warewulf/overlays/o2/rootfs/t2.txt": "A test file from o2", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "A test file from o1", + "t2.txt": "A test file from o2", + }, + }, + "template": { + node: node.Node{Profile: node.Profile{Comment: "A node comment"}}, + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/t1.txt.ww": "{{ .Comment }}", + "/image/t1.txt": "Previous content", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "A node comment", + "t1.txt.wwbackup": "Previous content", + }, + }, + "multifile": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/file.txt.ww": ` +{{- file "t1.txt" }} +T1 +{{ file "t2.txt" }} +T2 +{{ file "t3.txt" }} +T3 +`, + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "T1\n", + "t2.txt": "T2\n", + "t3.txt": "T3\n", + }, + }, + "symlink": { + overlays: []string{"o1"}, + overlaySymlinks: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/test-link": "/test-target", + }, + outputDir: "/image", + outputSymlinks: map[string]string{ + "test-link": "/test-target", + }, + }, + "symlink from template": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/test-link.ww": ` +{{- softlink "/test-target" }} +`, + }, + outputDir: "/image", + outputSymlinks: map[string]string{ + "test-link": "/test-target", + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + + for fileName, content := range tt.overlayFiles { + env.WriteFile(t, fileName, content) + } + for _, dirName := range tt.overlayDirs { + env.MkdirAll(t, dirName) + } + for linkName, target := range tt.overlaySymlinks { + env.Symlink(t, target, linkName) + } + + env.MkdirAll(t, tt.outputDir) + + assert.NoError(t, BuildOverlayIndir(tt.node, []node.Node{tt.node}, tt.overlays, env.GetPath(tt.outputDir))) + dirFiles := tt.outputDirs + for outputFile, _ := range tt.outputFiles { + dirFiles = append(dirFiles, outputFile) + } + for outputFile, _ := range tt.outputSymlinks { + dirFiles = append(dirFiles, outputFile) + } + sort.Strings(dirFiles) + assert.Equal(t, dirFiles, env.ReadDir(t, tt.outputDir)) + for fileName, content := range tt.outputFiles { + assert.Equal(t, content, env.ReadFile(t, path.Join(tt.outputDir, fileName))) + } + for _, dirName := range tt.outputDirs { + assert.True(t, util.IsDir(env.GetPath(path.Join(tt.outputDir, dirName))), fmt.Sprintf("%s is not a directory", dirName)) + } + for linkName, expTarget := range tt.outputSymlinks { + target, err := os.Readlink(env.GetPath(path.Join(tt.outputDir, linkName))) + assert.NoError(t, err) + assert.Equal(t, expTarget, target) + } + }) + } } func Test_BuildOverlay(t *testing.T) { - conf := warewulfconf.Get() - overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") - assert.NoError(t, overlayDirErr) - defer os.RemoveAll(overlayDir) - conf.Paths.WWOverlaydir = overlayDir - assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) - { - _, err := os.Create(path.Join(overlayDir, "o1", "o1.txt")) - assert.NoError(t, err) - } - assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - { - _, err := os.Create(path.Join(overlayDir, "o2", "o2.txt")) - assert.NoError(t, err) + var tests = []struct { + description string + nodeName string + context string + overlays []string + image string + contents []string + perms []int + hasFiles bool + }{ + { + description: "if no node, context, or overlays are specified then no overlay image is generated", + nodeName: "", + context: "", + overlays: nil, + image: "", + contents: nil, + }, + { + description: "if only node is specified then no overlay image is generated", + nodeName: "node1", + context: "", + overlays: nil, + image: "", + contents: nil, + }, + { + description: "if only context is specified then context named image is generated", + nodeName: "", + context: "system", + overlays: nil, + image: "__SYSTEM__.img", + contents: nil, + }, + { + description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory", + nodeName: "", + context: "", + overlays: []string{"o1"}, + image: "o1.img", + contents: []string{"o1.txt"}, + perms: []int{0644}, + }, + { + description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory", + nodeName: "", + context: "", + overlays: []string{"o1", "o2"}, + image: "o1-o2.img", + contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, + }, + { + description: "if a single node overlay is specified, then the overlay is built in a node overlay directory", + nodeName: "node1", + context: "", + overlays: []string{"o1"}, + image: "node1/o1.img", + contents: []string{"o1.txt"}, + perms: []int{0644}, + }, + { + description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory", + nodeName: "node1", + context: "", + overlays: []string{"o1", "o2"}, + image: "node1/o1-o2.img", + contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, + }, + { + description: "if no node system overlays are specified, then context pointed overlay is generated", + nodeName: "node1", + context: "system", + overlays: nil, + image: "node1/__SYSTEM__.img", + contents: nil, + }, + { + description: "if no node runtime overlays are specified, then context pointed overlay is generated", + nodeName: "node1", + context: "runtime", + overlays: nil, + image: "node1/__RUNTIME__.img", + contents: nil, + }, + { + description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory", + nodeName: "node1", + context: "system", + overlays: []string{"o1"}, + image: "node1/__SYSTEM__.img", + contents: []string{"o1.txt"}, + perms: []int{0644}, + }, + { + description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory", + nodeName: "node1", + context: "runtime", + overlays: []string{"o1"}, + image: "node1/__RUNTIME__.img", + contents: []string{"o1.txt"}, + perms: []int{0644}, + }, + { + description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays", + nodeName: "node1", + context: "system", + overlays: []string{"o1", "o2"}, + image: "node1/__SYSTEM__.img", + contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, + }, + { + description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays", + nodeName: "node1", + context: "runtime", + overlays: []string{"o1", "o2"}, + image: "node1/__RUNTIME__.img", + contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, + }, + { + description: "validating altered permissions are retained", + nodeName: "node1", + context: "runtime", + overlays: []string{"o3"}, + image: "node1/__RUNTIME__.img", + contents: []string{"subdir", "subdir/o3.txt"}, + perms: []int{0700, 0600}, + }, } - for _, tt := range buildOverlayTests { + env := testenv.New(t) + defer env.RemoveAll(t) + + env.CreateFile(t, "var/lib/warewulf/overlays/o1/rootfs/o1.txt") + env.CreateFile(t, "var/lib/warewulf/overlays/o2/rootfs/o2.txt") + env.CreateFile(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww") + env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir", 0700) + env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww", 0600) + + for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { - provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") - assert.NoError(t, provisionDirErr) - defer os.RemoveAll(provisionDir) - conf.Paths.WWProvisiondir = provisionDir - - err := BuildOverlay(nodeInfo, tt.context, tt.overlays) + err := BuildOverlay(nodeInfo, []node.Node{nodeInfo}, tt.context, tt.overlays) assert.NoError(t, err) if tt.image != "" { - image := path.Join(provisionDir, "overlays", tt.image) + image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image)) assert.FileExists(t, image) sort.Strings(tt.contents) - files, err := util.CpioFiles(image) + headers, err := readCpio(image) assert.NoError(t, err) - sort.Strings(files) - assert.Equal(t, tt.contents, files) + assert.Equal(t, len(tt.contents), len(headers)) + for i, file := range tt.contents { + assert.Equal(t, file, headers[file].Name) + if len(tt.perms) > i { + assert.Equal(t, tt.perms[i], int(headers[file].Mode.Perm())) + } + } } else { - dirName := path.Join(provisionDir, "overlays", tt.nodeName) + dirName := env.GetPath(path.Join("srv/warewulf/overlays", tt.nodeName)) isEmpty := dirIsEmpty(t, dirName) assert.True(t, isEmpty, "%v should be empty, but isn't", dirName) } @@ -269,83 +424,79 @@ func Test_BuildOverlay(t *testing.T) { } } -// Although these tests specify system and runtime overlays for the -// nodes, these overlays define the overlays that are defined in the -// configuration. BuildAllOverlays doesn't receive these as arguments, -// but builds all the overlays that are configured on the given node. -var buildAllOverlaysTests = []struct { - description string - nodes []string - systemOverlays [][]string - runtimeOverlays [][]string - createdOverlays []string -}{ - { - description: "empty input creates no overlays", - nodes: nil, - systemOverlays: nil, - runtimeOverlays: nil, - createdOverlays: nil, - }, - { - description: "a node with no overlays creates default system/runtime overlays", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: [][]string{{"o1"}}, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"}, - }, - { - description: "multiple nodes with no overlays creates default system/runtime overlays", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o2"}}, - runtimeOverlays: [][]string{{"o1"}, {"o2"}}, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"}, - }, - { - description: "a system overlay for a node generates a system overlay for that node", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: nil, - createdOverlays: []string{"node1/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with different system overlays generates a system overlay for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - runtimeOverlays: nil, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with a single runtime overlay generates a runtime overlay for the first node", - nodes: []string{"node1"}, - systemOverlays: nil, - runtimeOverlays: [][]string{{"o1"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz"}, - }, - { - description: "two nodes with different runtime overlays generates a system overlay for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: nil, - runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"}, - }, - { - description: "a node with both a runtime and system overlay generates an image for each", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: [][]string{{"o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with both runtime and system overlays generates each image for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - runtimeOverlays: [][]string{{"o2"}, {"o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"}, - }, -} - func Test_BuildAllOverlays(t *testing.T) { + var tests = []struct { + description string + nodes []string + systemOverlays [][]string + runtimeOverlays [][]string + createdOverlays []string + }{ + { + description: "empty input creates no overlays", + nodes: nil, + systemOverlays: nil, + runtimeOverlays: nil, + createdOverlays: nil, + }, + { + description: "a node with no overlays creates default system/runtime overlays", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: [][]string{{"o1"}}, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"}, + }, + { + description: "multiple nodes with no overlays creates default system/runtime overlays", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o2"}}, + runtimeOverlays: [][]string{{"o1"}, {"o2"}}, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"}, + }, + { + description: "a system overlay for a node generates a system overlay for that node", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: nil, + createdOverlays: []string{"node1/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with different system overlays generates a system overlay for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + runtimeOverlays: nil, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with a single runtime overlay generates a runtime overlay for the first node", + nodes: []string{"node1"}, + systemOverlays: nil, + runtimeOverlays: [][]string{{"o1"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz"}, + }, + { + description: "two nodes with different runtime overlays generates a system overlay for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: nil, + runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"}, + }, + { + description: "a node with both a runtime and system overlay generates an image for each", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: [][]string{{"o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with both runtime and system overlays generates each image for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + runtimeOverlays: [][]string{{"o2"}, {"o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"}, + }, + } + conf := warewulfconf.Get() overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") assert.NoError(t, overlayDirErr) @@ -354,7 +505,7 @@ func Test_BuildAllOverlays(t *testing.T) { assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - for _, tt := range buildAllOverlaysTests { + for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") assert.NoError(t, provisionDirErr) @@ -372,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes) + err := BuildAllOverlays(nodes, nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -385,65 +536,65 @@ func Test_BuildAllOverlays(t *testing.T) { } } -var buildSpecificOverlaysTests = []struct { - description string - nodes []string - overlays []string - images []string - succeed bool -}{ - { - description: "building no overlays for no nodes generates no error and no images", - nodes: nil, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building no overlays for a node generates no error and no images", - nodes: []string{"node1"}, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building no overlays for two nodes generates no error and no images", - nodes: []string{"node1", "node2"}, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building an overlay for a node generates an overlay image in that node's overlay directory", - nodes: []string{"node1"}, - overlays: []string{"o1"}, - images: []string{"node1/o1.img"}, - succeed: true, - }, - { - description: "building an overlay for two nodes generates an overlay image in each node's overlay directory", - nodes: []string{"node1", "node2"}, - overlays: []string{"o1"}, - images: []string{"node1/o1.img", "node2/o1.img"}, - succeed: true, - }, - { - description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory", - nodes: []string{"node1"}, - overlays: []string{"o1", "o2"}, - images: []string{"node1/o1.img", "node1/o2.img"}, - succeed: true, - }, - { - description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory", - nodes: []string{"node1", "node2"}, - overlays: []string{"o1", "o2"}, - images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"}, - succeed: true, - }, -} - func Test_BuildSpecificOverlays(t *testing.T) { + var tests = []struct { + description string + nodes []string + overlays []string + images []string + succeed bool + }{ + { + description: "building no overlays for no nodes generates no error and no images", + nodes: nil, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building no overlays for a node generates no error and no images", + nodes: []string{"node1"}, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building no overlays for two nodes generates no error and no images", + nodes: []string{"node1", "node2"}, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building an overlay for a node generates an overlay image in that node's overlay directory", + nodes: []string{"node1"}, + overlays: []string{"o1"}, + images: []string{"node1/o1.img"}, + succeed: true, + }, + { + description: "building an overlay for two nodes generates an overlay image in each node's overlay directory", + nodes: []string{"node1", "node2"}, + overlays: []string{"o1"}, + images: []string{"node1/o1.img", "node2/o1.img"}, + succeed: true, + }, + { + description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory", + nodes: []string{"node1"}, + overlays: []string{"o1", "o2"}, + images: []string{"node1/o1.img", "node1/o2.img"}, + succeed: true, + }, + { + description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory", + nodes: []string{"node1", "node2"}, + overlays: []string{"o1", "o2"}, + images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"}, + succeed: true, + }, + } + conf := warewulfconf.Get() overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") assert.NoError(t, overlayDirErr) @@ -452,7 +603,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - for _, tt := range buildSpecificOverlaysTests { + for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") assert.NoError(t, provisionDirErr) @@ -464,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays) + err := BuildSpecificOverlays(nodes, nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { @@ -493,3 +644,24 @@ func dirIsEmpty(t *testing.T, name string) bool { t.Log(dirnames) return false } + +func readCpio(name string) (headers map[string]*cpio.Header, err error) { + f, err := os.Open(name) + if err != nil { + return headers, err + } + defer f.Close() + + reader := cpio.NewReader(f) + headers = make(map[string]*cpio.Header) + for { + header, err := reader.Next() + if err == io.EOF { + return headers, nil + } + if err != nil { + return headers, err + } + headers[header.Name] = header + } +} diff --git a/internal/pkg/testenv/testenv.go b/internal/pkg/testenv/testenv.go index 115688e9..92642822 100644 --- a/internal/pkg/testenv/testenv.go +++ b/internal/pkg/testenv/testenv.go @@ -112,6 +112,14 @@ func (env *TestEnv) MkdirAll(t *testing.T, dirName string) { assert.NoError(t, err) } +// Chmod changes the mode of fileName in the test environment, which must already exist. +// +// Asserts no errors occur. +func (env *TestEnv) Chmod(t *testing.T, fileName string, mode int) { + err := os.Chmod(env.GetPath(fileName), os.FileMode(mode)) + assert.NoError(t, err) +} + // WriteFile writes content to fileName, creating any necessary // intermediate directories relative to the test environment. // @@ -140,12 +148,22 @@ func (env *TestEnv) ImportFile(t *testing.T, fileName string, inputFileName stri env.WriteFile(t, fileName, string(buffer)) } -// CreateFile creates an empty file to fileName, creating any necessary intermediate directories +// CreateFile creates an empty file at fileName, creating any necessary intermediate directories // relative to the test environment. func (env *TestEnv) CreateFile(t *testing.T, fileName string) { env.WriteFile(t, fileName, "") } +// Symlink creates a symlink at fileName to target, creating any necessary intermediate directories +// relative to the test environment. +func (env *TestEnv) Symlink(t *testing.T, target string, fileName string) { + dirName := filepath.Dir(fileName) + env.MkdirAll(t, dirName) + + err := os.Symlink(target, env.GetPath(fileName)) + assert.NoError(t, err) +} + // ReadFile returns the content of fileName as converted to a // string. // @@ -156,6 +174,20 @@ func (env *TestEnv) ReadFile(t *testing.T, fileName string) string { return string(buffer) } +// ReadDir returns the content of dirName as converted to a +// slice of strings. +// +// Asserts no errors occur. +func (env *TestEnv) ReadDir(t *testing.T, dirName string) []string { + entries, err := os.ReadDir(env.GetPath(dirName)) + assert.NoError(t, err) + var entryStrs []string + for _, entry := range entries { + entryStrs = append(entryStrs, entry.Name()) + } + return entryStrs +} + // RemoveAll deletes the temporary directory, and all its contents, // for the test environment. // diff --git a/internal/pkg/util/cpio.go b/internal/pkg/util/cpio.go deleted file mode 100644 index f8709c3d..00000000 --- a/internal/pkg/util/cpio.go +++ /dev/null @@ -1,31 +0,0 @@ -package util - -import ( - "io" - "os" - - "github.com/sassoftware/go-rpmutils/cpio" -) - -/* -Opens cpio archive and returns the file list -*/ -func CpioFiles(name string) (files []string, err error) { - f, err := os.Open(name) - if err != nil { - return files, err - } - defer f.Close() - - reader := cpio.NewReader(f) - for { - header, err := reader.Next() - if err == io.EOF { - return files, nil - } - if err != nil { - return files, err - } - files = append(files, header.Filename()) - } -} diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index bf57fed4..e037c343 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -18,10 +18,6 @@ import ( "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func BoolP(p *bool) bool { - return p != nil && *p -} - func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -136,37 +132,40 @@ func ValidString(pattern string, expr string) bool { return false } -// ****************************************************************************** func FindFiles(path string) []string { var ret []string - wwlog.Debug("Changing directory to FindFiles path: %s", path) - err := os.Chdir(path) - if err != nil { - wwlog.Warn("Could not chdir() to: %s", path) - return ret - } + wwlog.Debug("Finding files in path: %s", path) - err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + err := filepath.Walk(path, func(location string, info os.FileInfo, err error) error { if err != nil { + wwlog.Warn("Error walking path %s: %v", location, err) return err } - if location == "." { + // Get the relative path from the base directory + relPath, relErr := filepath.Rel(path, location) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", location, relErr) + return relErr + } + + if relPath == "." { return nil } - if IsDir(location) { - wwlog.Debug("FindFiles() found directory: %s", location) - ret = append(ret, location+"/") + if info.IsDir() { + wwlog.Debug("FindFiles() found directory: %s", relPath) + ret = append(ret, relPath+"/") } else { - wwlog.Debug("FindFiles() found file: %s", location) - ret = append(ret, location) + wwlog.Debug("FindFiles() found file: %s", relPath) + ret = append(ret, relPath) } return nil }) if err != nil { + wwlog.Warn("Error during file walk: %v", err) return ret } @@ -184,82 +183,100 @@ func FindFilterFiles( ignorePattern []string, ignore_xdev bool) (ofiles []string, err error) { wwlog.Debug("Finding files: %s include: %s ignore: %s", path, includePattern, ignorePattern) - // preprocess patterns to remove leading (and trailing) /, as we are handling relative paths + + // Preprocess patterns to remove leading (and trailing) /, as we are handling relative paths for i, pattern := range ignorePattern { ignorePattern[i] = strings.Trim(pattern, "/") } - cwd, err := os.Getwd() + + // Convert the base path to an absolute path + absPath, err := filepath.Abs(path) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to resolve absolute path: %s: %w", path, err) } - defer func() { - err = FirstError(err, os.Chdir(cwd)) - }() - err = os.Chdir(path) - if err != nil { - return ofiles, fmt.Errorf("failed to change path: %s: %w", path, err) - } - // expand our include list as fspath.Match with /foo/* would catch /foo/baar but - // not /foo/baar/sibling + + // Expand the include list var globedInclude []string for _, include := range includePattern { - globed, err := filepath.Glob(include) + globed, err := filepath.Glob(filepath.Join(absPath, include)) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to glob pattern %s: %w", include, err) } globedInclude = append(globedInclude, globed...) } + + var dev uint64 if ignore_xdev { wwlog.Debug("Ignoring cross-device (xdev) files") + pathStat, err := os.Stat(absPath) + if err != nil { + return ofiles, fmt.Errorf("failed to stat base path: %s: %w", absPath, err) + } + dev = pathStat.Sys().(*syscall.Stat_t).Dev } - path_stat, err := os.Stat(".") - if err != nil { - return ofiles, err - } - - dev := path_stat.Sys().(*syscall.Stat_t).Dev for _, inc := range globedInclude { - wwlog.Debug("inc %s", inc) + wwlog.Debug("Processing include pattern: %s", inc) stat, err := os.Lstat(inc) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to stat include: %s: %w", inc, err) } + if stat.IsDir() { - // get the rest of dir + // Walk the directory err = filepath.WalkDir(inc, func(location string, info fs.DirEntry, err error) error { if err != nil { return err } - if location == "." { + + relPath, relErr := filepath.Rel(absPath, location) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", location, relErr) + return relErr + } + + if relPath == "." { return nil } + fsInfo, err := info.Info() if err != nil { return err } + if ignore_xdev && fsInfo.Sys().(*syscall.Stat_t).Dev != dev { wwlog.Debug("Ignored (cross-device): %s", location) return nil } - for _, ignored_pat := range ignorePattern { - if ignored, _ := filepath.Match(ignored_pat, location); ignored { - wwlog.Debug("Ignored %s due to pattern %s", location, ignored_pat) - return filepath.SkipDir + + for _, ignoredPattern := range ignorePattern { + if ignored, _ := filepath.Match(ignoredPattern, relPath); ignored { + wwlog.Debug("Ignored %s due to pattern %s", relPath, ignoredPattern) + if info.IsDir() { + return filepath.SkipDir + } + return nil } } - ofiles = append(ofiles, location) + + ofiles = append(ofiles, relPath) return nil }) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("error walking directory %s: %w", inc, err) } } else { - ofiles = append(ofiles, inc) + // Add the file directly + relPath, relErr := filepath.Rel(absPath, inc) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", inc, relErr) + return ofiles, relErr + } + ofiles = append(ofiles, relPath) } } - return ofiles, err + return ofiles, nil } // ****************************************************************************** @@ -346,6 +363,7 @@ func AppendLines(fileName string, lines []string) error { Create an archive using cpio */ func CpioCreate( + rootdir string, ifiles []string, ofile string, format string, @@ -354,7 +372,8 @@ func CpioCreate( args := []string{ "--quiet", "--create", - "-H", format, + "--directory", rootdir, + "--format", format, "--file=" + ofile} args = append(args, cpio_args...) @@ -492,21 +511,9 @@ func BuildFsImage( return fmt.Errorf("failed to create image directory for %s: %s: %w", name, imagePath, err) } wwlog.Debug("Created image directory for %s: %s", name, imagePath) - cwd, err := os.Getwd() - if err != nil { - return err - } - defer func() { - err = FirstError(err, os.Chdir(cwd)) - }() - err = os.Chdir(rootfsPath) - if err != nil { - return fmt.Errorf("failed chdir to fs directory for %s: %s: %w", name, rootfsPath, err) - } - wwlog.Verbose("changed to: %s", rootfsPath) files, err := FindFilterFiles( - ".", + rootfsPath, include, ignore, ignore_xdev) @@ -515,6 +522,7 @@ func BuildFsImage( } err = CpioCreate( + rootfsPath, files, imagePath, format, diff --git a/internal/pkg/util/util_test.go b/internal/pkg/util/util_test.go index e5087212..b81710cc 100644 --- a/internal/pkg/util/util_test.go +++ b/internal/pkg/util/util_test.go @@ -1,53 +1,94 @@ package util import ( - "os" "path/filepath" - "reflect" - "sort" "testing" "github.com/stretchr/testify/assert" - "github.com/warewulf/warewulf/internal/pkg/wwlog" + "github.com/warewulf/warewulf/internal/pkg/testenv" ) -func TryCreatePath(t *testing.T, elem ...string) { - err := os.MkdirAll(filepath.Join(elem...), os.ModePerm) - if err != nil { - t.Errorf("Failed creating dir: %v", err) - t.FailNow() +func Test_FindFiles(t *testing.T) { + var tests = map[string]struct { + createFiles []string + findFiles []string + }{ + "no files": { + createFiles: []string{}, + findFiles: nil, + }, + "single file": { + createFiles: []string{"testfile"}, + findFiles: []string{"testfile"}, + }, + "nested file": { + createFiles: []string{"testdir/testfile"}, + findFiles: []string{"testdir/", "testdir/testfile"}, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + env.MkdirAll(t, "/test") + for _, file_ := range tt.createFiles { + env.CreateFile(t, filepath.Join("/test", file_)) + } + + files := FindFiles(env.GetPath("/test")) + assert.Equal(t, tt.findFiles, files) + }) } } func Test_FindFilterFiles(t *testing.T) { - wwlog.SetLogLevel(wwlog.DEBUG) - dir, err := os.MkdirTemp(os.TempDir(), "warewulf-test") - if err != nil { - t.Errorf("Failed creating tmpdir: %v", err) - t.FailNow() - } - defer os.RemoveAll(dir) - TryCreatePath(t, dir, "boot") - TryCreatePath(t, dir, "usr", "local") - TryCreatePath(t, dir, "usr", "bin") - TryCreatePath(t, dir, "usr", "usr", "local") - TryCreatePath(t, dir, "bin") - TryCreatePath(t, dir, "lib") - - assert.NoError(t, os.Symlink("/path/to/target", filepath.Join(dir, "symlink"))) - - files, err := FindFilterFiles(dir, []string{"boot", "usr", "bin", "symlink"}, []string{"/b*/", "/usr/local"}, true) - - if err != nil { - t.Errorf("FindFilerFiles failed: %v", err) - t.FailNow() + var tests = map[string]struct { + createFiles []string + include []string + exclude []string + findFiles []string + }{ + "no files": { + createFiles: []string{}, + include: []string{"*"}, + findFiles: nil, + }, + "single file": { + createFiles: []string{"testfile"}, + include: []string{"*"}, + findFiles: []string{"testfile"}, + }, + "nested file": { + createFiles: []string{"testdir/testfile"}, + include: []string{"*"}, + findFiles: []string{"testdir", "testdir/testfile"}, + }, + "multiple files": { + createFiles: []string{"test1/testfile", "test2/testfile"}, + include: []string{"*"}, + findFiles: []string{"test1", "test1/testfile", "test2", "test2/testfile"}, + }, + "excluded files": { + createFiles: []string{"test1/test1", "test1/test2", "test2/test1", "test2/test2"}, + include: []string{"*"}, + exclude: []string{"test1/*2", "test2"}, + findFiles: []string{"test1", "test1/test1"}, + }, } - expected := []string{"usr", "usr/bin", "usr/usr", "usr/usr/local", "symlink"} - sort.Strings(expected) - sort.Strings(files) - if !reflect.DeepEqual(files, expected) { - t.Errorf("expected %v, got %v", expected, files) - t.FailNow() + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + env.MkdirAll(t, "/test") + for _, file_ := range tt.createFiles { + env.CreateFile(t, filepath.Join("/test", file_)) + } + + files, err := FindFilterFiles(env.GetPath("/test"), tt.include, tt.exclude, true) + assert.NoError(t, err) + assert.Equal(t, tt.findFiles, files) + }) } } diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index a40c1781..919ee05e 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -45,9 +45,7 @@ func sendFile( } func getOverlayFile(n node.Node, context string, stage_overlays []string, autobuild bool) (stage_file string, err error) { - stage_file = overlay.OverlayImage(n.Id(), context, stage_overlays) - err = nil build := !util.IsFile(stage_file) wwlog.Verbose("stage file: %s", stage_file) if !build && autobuild { @@ -60,10 +58,23 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu } if build { + registry, err := node.New() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays) + err = overlay.BuildSpecificOverlays([]node.Node{n}, allNodes, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}) + err = overlay.BuildAllOverlays([]node.Node{n}, allNodes, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", diff --git a/userdocs/contents/overlays.rst b/userdocs/contents/overlays.rst index a71439be..0327dc9c 100644 --- a/userdocs/contents/overlays.rst +++ b/userdocs/contents/overlays.rst @@ -224,24 +224,13 @@ built. Specific overlays can be selected with ``-O`` flag. For debugging purposes the templates can be written to a directory given via the ``-o`` flag. -On clusters with large numbers of nodes a significant speedup can be achieved -by building overlays in parallel. Adding parallel overlay building to `wwctl` -is planned, see issue `#1087 `_. -Until parallel overlay building is implemented, builds can be easily done in -parallel with Gnu `parallel` or `xargs`, for example: +Overlay images for multiple node are built in parallel. By default, each CPU in +the Warewulf server will build overlays independently. The number of workers +can be specified with the ``--workers`` option. -.. code-block:: console - - # Gnu parallel - wwctl node list | awk '{print $1}' | grep -v "NODE " | parallel -j 12 \ - wwctl overlay build -N {} - - # xargs - wwctl node list | awk '{print $1}' | grep -v "NODE " | xargs -n 1 -P 12 \ - wwctl overlay build -N - -By default Warewulf will build/update and cache overlays as needed -(configurable in the ``warewulf.conf``). +Warewulf will attempt to build/update overlays as needed +(configurable in the ``warewulf.conf``); but not all cases are detected, +and manual overlay builds are often necessary. Chmod -----