kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1604 from anderbubble/parallel-overlay-build

Browse Source 
Parallel and optimized overlay builds
This commit is contained in:
Jonathon Anderson
2025-01-08 11:13:01 -07:00
committed by GitHub
parent e64396cfab c244adc7f6
commit baf8124186
24 changed files with 894 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@@ -45,7 +45,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Bump github.com/spf13/cobra from 1.7.0 to 1.8.0 #1166
- Bump github.com/fatih/color from 1.15.0 to 1.17.0 #1224
- Bump github.com/coreos/ignition/v2 from 2.15.0 to 2.19.0 #1239
- Bump github.com/sassoftware/go-rpmutils from 0.2.0 to 0.4.0 #1217
- Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 #1481
- Bump google.golang.org/protobuf from 1.34.1 to 1.35.1 #1480
- Bump golang.org/x/term from 0.20.0 to 0.25.0 #1476
@@ -85,6 +84,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Added note to booting userdoc for removing machine-id. #1609
- Log cpio errors more prominently. #1615
- Improved syncuser conflict help text. #1614
- Parallelized overlay build. #1018
- Parallelized and optimized overlay build. #1018
### Removed

2
go.mod
View File

@@ -5,6 +5,7 @@ go 1.21.0
require (
dario.cat/mergo v1.0.1
github.com/Masterminds/sprig/v3 v3.3.0
github.com/cavaliergopher/cpio v1.0.1
github.com/cheynewallace/tabby v1.1.1
github.com/containers/image/v5 v5.32.2
github.com/containers/storage v1.55.2
@@ -21,7 +22,6 @@ require (
github.com/opencontainers/image-spec v1.1.0
github.com/opencontainers/umoci v0.4.7
github.com/pkg/errors v0.9.1
github.com/sassoftware/go-rpmutils v0.4.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
github.com/talos-systems/go-smbios v0.1.1

4
go.sum
View File

@@ -32,6 +32,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM=
github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -310,8 +312,6 @@ github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZP
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA=
github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=

12
internal/app/wwctl/container/exec/child/main.go
View File

@@ -97,17 +97,17 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) {
return fmt.Errorf("could not open node configuration: %s", err)
}
nodes, err := nodeDB.FindAllNodes()
allNodes, err := nodeDB.FindAllNodes()
if err != nil {
return fmt.Errorf("could not get node list: %s", err)
}
nodes = node.FilterNodeListByName(nodes, []string{nodename})
if len(nodes) != 1 {
filteredNodes := node.FilterNodeListByName(allNodes, []string{nodename})
if len(filteredNodes) != 1 {
return fmt.Errorf("no single node idendified with %s", nodename)
}
overlays := nodes[0].SystemOverlay
overlays = append(overlays, nodes[0].RuntimeOverlay...)
err = overlay.BuildOverlayIndir(nodes[0], overlays, path.Join(runDir, "nodeoverlay"))
overlays := filteredNodes[0].SystemOverlay
overlays = append(overlays, filteredNodes[0].RuntimeOverlay...)
err = overlay.BuildOverlayIndir(filteredNodes[0], allNodes, overlays, path.Join(runDir, "nodeoverlay"))
if err != nil {
return fmt.Errorf("could not build overlay: %s", err)
}

20
internal/app/wwctl/overlay/build/main.go
View File

@@ -18,18 +18,21 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
return fmt.Errorf("could not open node configuration: %s", err)
}
db, err := nodeDB.FindAllNodes()
allNodes, err := nodeDB.FindAllNodes()
if err != nil {
return fmt.Errorf("could not get node list: %s", err)
}
var filteredNodes []node.Node
if len(args) > 0 {
args = hostlist.Expand(args)
db = node.FilterNodeListByName(db, args)
filteredNodes = node.FilterNodeListByName(allNodes, args)
if len(db) < len(args) {
if len(filteredNodes) < len(args) {
return errors.New("failed to find nodes")
}
} else {
filteredNodes = allNodes
}
// NOTE: this is to keep backward compatible
@@ -49,26 +52,25 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
}
if len(args) > 0 {
if len(db) != 1 {
if len(filteredNodes) != 1 {
return errors.New("must specify one node to build overlay")
}
for _, node := range db {
return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir)
for _, node := range filteredNodes {
return overlay.BuildOverlayIndir(node, allNodes, OverlayNames, OverlayDir)
}
} else {
return errors.New("must specify a node to build overlay")
}
}
oldMask := syscall.Umask(007)
defer syscall.Umask(oldMask)
if len(OverlayNames) > 0 {
err = overlay.BuildSpecificOverlays(db, OverlayNames)
err = overlay.BuildSpecificOverlays(filteredNodes, allNodes, OverlayNames, Workers)
} else {
err = overlay.BuildAllOverlays(db)
err = overlay.BuildAllOverlays(filteredNodes, allNodes, Workers)
}
if err != nil {

4
internal/app/wwctl/overlay/build/root.go
View File

@@ -2,6 +2,7 @@ package build
import (
"log"
"runtime"
"github.com/spf13/cobra"
"github.com/warewulf/warewulf/internal/pkg/node"
@@ -31,6 +32,7 @@ var (
}
OverlayNames []string
OverlayDir string
Workers int
)
func init() {
@@ -44,7 +46,7 @@ func init() {
}
baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to
the given directory. An overlay must also be ge given to use this option.`)
baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays")
}
// GetRootCommand returns the root cobra.Command for the application.

2
internal/app/wwctl/overlay/imprt/main.go
View File

@@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) {
}
}
return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName})
return overlay.BuildSpecificOverlays(updateNodes, nodes, []string{overlayName}, Workers)
}
return nil

4
internal/app/wwctl/overlay/imprt/root.go
View File

@@ -1,6 +1,8 @@
package imprt
import (
"runtime"
"github.com/spf13/cobra"
"github.com/warewulf/warewulf/internal/pkg/overlay"
)
@@ -24,11 +26,13 @@ var (
}
NoOverlayUpdate bool
CreateDirs bool
Workers int
)
func init() {
baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays")
baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories")
baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays")
}
// GetRootCommand returns the root cobra.Command for the application.

7
internal/app/wwctl/overlay/show/main.go
View File

@@ -57,7 +57,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
nodeConf = node.NewNode(hostName)
nodeConf.ClusterName = hostName
}
tstruct, err := overlay.InitStruct(overlayName, nodeConf)
var allNodes []node.Node
allNodes, err = nodeDB.FindAllNodes()
if err != nil {
return err
}
tstruct, err := overlay.InitStruct(overlayName, nodeConf, allNodes)
if err != nil {
return err
}

14
internal/pkg/config/buildconfig.go.in
View File

@@ -2,8 +2,6 @@ package config
import (
"path"
"github.com/warewulf/warewulf/internal/pkg/util"
)
var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf"
@@ -36,7 +34,7 @@ type TFTPConf struct {
}
func (this TFTPConf) Enabled() bool {
return util.BoolP(this.EnabledP)
return BoolP(this.EnabledP)
}
// WarewulfConf adds additional Warewulf-specific configuration to
@@ -52,23 +50,23 @@ type WarewulfConf struct {
}
func (this WarewulfConf) Secure() bool {
return util.BoolP(this.SecureP)
return BoolP(this.SecureP)
}
func (this WarewulfConf) AutobuildOverlays() bool {
return util.BoolP(this.AutobuildOverlaysP)
return BoolP(this.AutobuildOverlaysP)
}
func (this WarewulfConf) EnableHostOverlay() bool {
return util.BoolP(this.EnableHostOverlayP)
return BoolP(this.EnableHostOverlayP)
}
func (this WarewulfConf) Syslog() bool {
return util.BoolP(this.SyslogP)
return BoolP(this.SyslogP)
}
func (this WarewulfConf) GrubBoot() bool {
return util.BoolP(this.GrubBootP)
return BoolP(this.GrubBootP)
}
func (paths BuildConfig) NodesConf() string {

6
internal/pkg/config/dhcp.go
View File

@@ -1,9 +1,5 @@
package config
import (
"github.com/warewulf/warewulf/internal/pkg/util"
)
// DHCPConf represents the configuration for the DHCP service that
// Warewulf will configure.
type DHCPConf struct {
@@ -15,5 +11,5 @@ type DHCPConf struct {
}
func (this DHCPConf) Enabled() bool {
return util.BoolP(this.EnabledP)
return BoolP(this.EnabledP)
}

8
internal/pkg/config/mounts.go
View File

@@ -1,9 +1,5 @@
package config
import (
"github.com/warewulf/warewulf/internal/pkg/util"
)
// A MountEntry represents a bind mount that is applied to a container
// during exec and shell.
type MountEntry struct {
@@ -15,9 +11,9 @@ type MountEntry struct {
}
func (this MountEntry) ReadOnly() bool {
return util.BoolP(this.ReadOnlyP)
return BoolP(this.ReadOnlyP)
}
func (this MountEntry) Copy() bool {
return util.BoolP(this.CopyP)
return BoolP(this.CopyP)
}

6
internal/pkg/config/nfs.go
View File

@@ -2,8 +2,6 @@ package config
import (
"github.com/creasty/defaults"
"github.com/warewulf/warewulf/internal/pkg/util"
)
// NFSConf represents the NFS configuration that will be used by
@@ -16,7 +14,7 @@ type NFSConf struct {
}
func (this NFSConf) Enabled() bool {
return util.BoolP(this.EnabledP)
return BoolP(this.EnabledP)
}
// An NFSExportConf reprents a single NFS export / mount.
@@ -28,7 +26,7 @@ type NFSExportConf struct {
}
func (this NFSExportConf) Mount() bool {
return util.BoolP(this.MountP)
return BoolP(this.MountP)
}
// Implements the Unmarshal interface for NFSConf to set default

5
internal/pkg/config/util.go Normal file
View File

@@ -0,0 +1,5 @@
package config
func BoolP(p *bool) bool {
return p != nil && *p
}

12
internal/pkg/configure/hostfile.go
View File

@@ -20,8 +20,18 @@ func Hostfile() (err error) {
return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay")
}
var allNodes []node.Node
if nodeDB, err := node.New(); err != nil {
return err
} else {
allNodes, err = nodeDB.FindAllNodes()
if err != nil {
return err
}
}
hostname, _ := os.Hostname()
tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname))
tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname), allNodes)
if err != nil {
return err
}

17
internal/pkg/overlay/datastructure.go
View File

@@ -46,16 +46,12 @@ type TemplateStruct struct {
/*
Initialize an TemplateStruct with the given node.NodeInfo
*/
func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) {
func InitStruct(overlayName string, nodeData node.Node, allNodes []node.Node) (TemplateStruct, error) {
var tstruct TemplateStruct
tstruct.Overlay = overlayName
hostname, _ := os.Hostname()
tstruct.BuildHost = hostname
controller := warewulfconf.Get()
nodeDB, err := node.New()
if err != nil {
return tstruct, err
}
tstruct.ThisNode = &nodeData
if tstruct.ThisNode.Kernel == nil {
tstruct.ThisNode.Kernel = new(node.KernelConf)
@@ -75,10 +71,6 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error)
tstruct.Ipaddr6 = controller.Ipaddr6
tstruct.Netmask = controller.Netmask
tstruct.Network = controller.Network
allNodes, err := nodeDB.FindAllNodes()
if err != nil {
return tstruct, err
}
// init some convenience vars
tstruct.Id = nodeData.Id()
tstruct.Hostname = nodeData.Id()
@@ -92,14 +84,11 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error)
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
dec := gob.NewDecoder(&buf)
err = enc.Encode(nodeData)
if err != nil {
if err := enc.Encode(nodeData); err != nil {
return tstruct, err
}
err = dec.Decode(&tstruct)
if err != nil {
if err := dec.Decode(&tstruct); err != nil {
return tstruct, err
}
return tstruct, nil
}

305
internal/pkg/overlay/overlay.go
View File

@@ -10,11 +10,10 @@ import (
"path/filepath"
"regexp"
"strings"
"syscall"
"sync"
"text/template"
"github.com/Masterminds/sprig/v3"
"github.com/pkg/errors"
"github.com/warewulf/warewulf/internal/pkg/config"
"github.com/warewulf/warewulf/internal/pkg/node"
@@ -23,7 +22,7 @@ import (
)
var (
ErrDoesNotExist = errors.New("overlay does not exist")
ErrDoesNotExist = fmt.Errorf("overlay does not exist")
)
// Overlay represents an overlay directory path.
@@ -102,40 +101,76 @@ func (this Overlay) IsDistributionOverlay() bool {
return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir()
}
/*
Build all overlays for a node
*/
func BuildAllOverlays(nodes []node.Node) error {
func BuildAllOverlays(nodes []node.Node, allNodes []node.Node, workerCount int) error {
nodeChan := make(chan node.Node, len(nodes))
errChan := make(chan error, len(nodes)*2)
var wg sync.WaitGroup
worker := func() {
for n := range nodeChan {
wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", "))
if err := BuildOverlay(n, allNodes, "system", n.SystemOverlay); err != nil {
errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err)
}
wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", "))
if err := BuildOverlay(n, allNodes, "runtime", n.RuntimeOverlay); err != nil {
errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err)
}
}
wg.Done()
}
for i := 0; i < workerCount; i++ {
wg.Add(1)
go worker()
}
for _, n := range nodes {
nodeChan <- n
}
close(nodeChan)
sysOverlays := n.SystemOverlay
wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(sysOverlays, ", "))
err := BuildOverlay(n, "system", sysOverlays)
if err != nil {
return fmt.Errorf("could not build system overlays %v for node %s: %w", sysOverlays, n.Id(), err)
}
runOverlays := n.RuntimeOverlay
wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(runOverlays, ", "))
err = BuildOverlay(n, "runtime", runOverlays)
if err != nil {
return fmt.Errorf("could not build runtime overlays %v for node %s: %w", runOverlays, n.Id(), err)
}
wg.Wait()
close(errChan)
for err := range errChan {
return err
}
return nil
}
// TODO: Add an Overlay Delete for both sourcedir and image
func BuildSpecificOverlays(nodes []node.Node, allNodes []node.Node, overlayNames []string, workerCount int) error {
nodeChan := make(chan node.Node, len(nodes))
errChan := make(chan error, len(nodes))
func BuildSpecificOverlays(nodes []node.Node, overlayNames []string) error {
for _, n := range nodes {
wwlog.Info("Building overlay for %s: %v", n, overlayNames)
for _, overlayName := range overlayNames {
err := BuildOverlay(n, "", []string{overlayName})
if err != nil {
return fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err)
var wg sync.WaitGroup
worker := func() {
for n := range nodeChan {
wwlog.Info("Building overlay for %s: %v", n, overlayNames)
for _, overlayName := range overlayNames {
err := BuildOverlay(n, allNodes, "", []string{overlayName})
if err != nil {
errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err)
}
}
}
wg.Done()
}
for i := 0; i < workerCount; i++ {
wg.Add(1)
go worker()
}
for _, n := range nodes {
nodeChan <- n
}
close(nodeChan)
wg.Wait()
close(errChan)
for err := range errChan {
return err
}
return nil
}
@@ -155,7 +190,16 @@ func BuildHostOverlay() error {
if !(stats.Mode() == os.FileMode(0750|os.ModeDir) || stats.Mode() == os.FileMode(0700|os.ModeDir)) {
wwlog.SecWarn("Permissions of host overlay dir %s are %s (750 is considered as secure)", hostdir, stats.Mode())
}
return BuildOverlayIndir(hostData, []string{"host"}, "/")
registry, err := node.New()
if err != nil {
return err
}
var allNodes []node.Node
allNodes, err = registry.FindAllNodes()
if err != nil {
return err
}
return BuildOverlayIndir(hostData, allNodes, []string{"host"}, "/")
}
/*
@@ -179,14 +223,13 @@ func FindOverlays() (overlayList []string, err error) {
overlayList = append(overlayList, file.Name())
}
}
return overlayList, nil
}
/*
Build the given overlays for a node and create a Image for them
*/
func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) error {
func BuildOverlay(nodeConf node.Node, allNodes []node.Node, context string, overlayNames []string) error {
if len(overlayNames) == 0 && context == "" {
return nil
}
@@ -211,7 +254,7 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err
wwlog.Debug("Created temporary directory for %s: %s", name, buildDir)
err = BuildOverlayIndir(nodeConf, overlayNames, buildDir)
err = BuildOverlayIndir(nodeConf, allNodes, overlayNames, buildDir)
if err != nil {
return fmt.Errorf("failed to generate files for %s: %w", name, err)
}
@@ -231,155 +274,157 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err
return err
}
/*
Build the given overlays for a node in the given directory. If the given does not
exists it will be created.
*/
func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error {
var regFile *regexp.Regexp
var regLink *regexp.Regexp
func init() {
regFile = regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`)
regLink = regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`)
}
// Build the given overlays for a node in the given directory.
func BuildOverlayIndir(nodeData node.Node, allNodes []node.Node, overlayNames []string, outputDir string) error {
if len(overlayNames) == 0 {
return nil
}
if !util.IsDir(outputDir) {
return errors.Errorf("output must a be a directory: %s", outputDir)
return fmt.Errorf("output must a be a directory: %s", outputDir)
}
if !util.ValidString(strings.Join(overlayNames, ""), "^[a-zA-Z0-9-._:]+$") {
return errors.Errorf("overlay names contains illegal characters: %v", overlayNames)
return fmt.Errorf("overlay names contains illegal characters: %v", overlayNames)
}
// Temporarily set umask to 0000, so directories in the overlay retain permissions
defer syscall.Umask(syscall.Umask(0))
wwlog.Verbose("Processing node/overlay: %s/%s", nodeData.Id(), strings.Join(overlayNames, "-"))
wwlog.Verbose("Processing node/overlays: %s/%s", nodeData.Id(), strings.Join(overlayNames, ","))
for _, overlayName := range overlayNames {
wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir)
overlaySourceDir := GetOverlay(overlayName).Rootfs()
wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir)
err := os.Chdir(overlaySourceDir)
if err != nil {
return fmt.Errorf("directory: %s name: %s err: %w", overlaySourceDir, overlayName, ErrDoesNotExist)
overlayRootfs := GetOverlay(overlayName).Rootfs()
if !util.IsDir(overlayRootfs) {
return fmt.Errorf("overlay %s: %w", overlayName, ErrDoesNotExist)
}
wwlog.Verbose("Walking the overlay structure: %s", overlaySourceDir)
err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error {
wwlog.Debug("Walking the overlay structure: %s", overlayRootfs)
err := filepath.Walk(overlayRootfs, func(walkPath string, info os.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("error for %s: %w", location, err)
return fmt.Errorf("error for %s: %w", walkPath, err)
}
wwlog.Debug("Found overlay file: %s", walkPath)
wwlog.Debug("Found overlay file: %s", location)
relPath, relErr := filepath.Rel(overlayRootfs, walkPath)
if relErr != nil {
wwlog.Warn("Error computing relative path for %s: %v", walkPath, relErr)
return relErr
}
outputPath := path.Join(outputDir, relPath)
if info.IsDir() {
wwlog.Debug("Found directory: %s", location)
wwlog.Debug("Found directory: %s", walkPath)
err = os.MkdirAll(path.Join(outputDir, location), info.Mode())
if err != nil {
if err = os.MkdirAll(outputPath, info.Mode()); err != nil {
return fmt.Errorf("could not create directory within overlay: %w", err)
}
err = util.CopyUIDGID(location, path.Join(outputDir, location))
if err != nil {
if err = util.CopyUIDGID(walkPath, outputPath); err != nil {
return fmt.Errorf("failed setting permissions on overlay directory: %w", err)
}
wwlog.Debug("Created directory in overlay: %s", location)
wwlog.Debug("Created directory in overlay: %s", outputPath)
} else if filepath.Ext(location) == ".ww" {
tstruct, err := InitStruct(overlayName, nodeData)
} else if filepath.Ext(walkPath) == ".ww" {
originalOutputPath := outputPath
outputPath := strings.TrimSuffix(outputPath, ".ww")
tstruct, err := InitStruct(overlayName, nodeData, allNodes)
if err != nil {
return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err)
}
tstruct.BuildSource = path.Join(overlaySourceDir, location)
wwlog.Verbose("Evaluating overlay template file: %s", location)
destFile := strings.TrimSuffix(location, ".ww")
tstruct.BuildSource = walkPath
wwlog.Verbose("Evaluating overlay template file: %s", walkPath)
buffer, backupFile, writeFile, err := RenderTemplateFile(location, tstruct)
buffer, backupFile, writeFile, err := RenderTemplateFile(walkPath, tstruct)
if err != nil {
return fmt.Errorf("failed to render template %s: %w", location, err)
return fmt.Errorf("failed to render template %s: %w", walkPath, err)
}
if writeFile {
destFileName := destFile
var fileBuffer bytes.Buffer
// search for magic file name comment
fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes()))
fileScanner.Split(ScanLines)
regFile := regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`)
regLink := regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`)
foundFileComment := false
for fileScanner.Scan() {
line := fileScanner.Text()
filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1)
softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1)
if len(softlinkFromTemplate) != 0 {
wwlog.Debug("Creating soft link %s -> %s", destFileName, softlinkFromTemplate[0][1])
return os.Symlink(softlinkFromTemplate[0][1], path.Join(outputDir, destFileName))
} else if len(filenameFromTemplate) != 0 {
wwlog.Debug("Writing file %s", filenameFromTemplate[0][1])
if foundFileComment {
err = CarefulWriteBuffer(path.Join(outputDir, destFileName),
fileBuffer, backupFile, info.Mode())
if err != nil {
return fmt.Errorf("could not write file from template: %w", err)
}
err = util.CopyUIDGID(location, path.Join(outputDir, destFileName))
if err != nil {
return fmt.Errorf("failed setting permissions on template output file: %w", err)
}
fileBuffer.Reset()
if !writeFile {
return nil
}
var fileBuffer bytes.Buffer
// search for magic file name comment
fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes()))
fileScanner.Split(ScanLines)
foundFileComment := false
for fileScanner.Scan() {
line := fileScanner.Text()
filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1)
softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1)
if len(softlinkFromTemplate) != 0 {
wwlog.Debug("Creating soft link %s -> %s", outputPath, softlinkFromTemplate[0][1])
return os.Symlink(softlinkFromTemplate[0][1], outputPath)
} else if len(filenameFromTemplate) != 0 {
wwlog.Debug("Writing file %s", filenameFromTemplate[0][1])
if foundFileComment {
err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode())
if err != nil {
return fmt.Errorf("could not write file from template: %w", err)
}
destFileName = path.Join(path.Dir(destFile), filenameFromTemplate[0][1])
foundFileComment = true
} else {
_, _ = fileBuffer.WriteString(line)
err = util.CopyUIDGID(walkPath, outputPath)
if err != nil {
return fmt.Errorf("failed setting permissions on template output file: %w", err)
}
fileBuffer.Reset()
}
outputPath = path.Join(path.Dir(originalOutputPath), filenameFromTemplate[0][1])
foundFileComment = true
} else {
if _, err = fileBuffer.WriteString(line); err != nil {
return fmt.Errorf("could not write to template buffer: %w", err)
}
}
err = CarefulWriteBuffer(path.Join(outputDir, destFileName), fileBuffer, backupFile, info.Mode())
if err != nil {
return fmt.Errorf("could not write file from template: %w", err)
}
err = util.CopyUIDGID(location, path.Join(outputDir, destFileName))
if err != nil {
return fmt.Errorf("failed setting permissions on template output file: %w", err)
}
wwlog.Debug("Wrote template file into overlay: %s", destFile)
// } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b {
// wwlog.Debug("Ignoring WW template file: %s", location)
}
err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode())
if err != nil {
return fmt.Errorf("could not write file from template: %w", err)
}
err = util.CopyUIDGID(walkPath, outputPath)
if err != nil {
return fmt.Errorf("failed setting permissions on template output file: %w", err)
}
wwlog.Debug("Wrote template file into overlay: %s", outputPath)
} else if info.Mode()&os.ModeSymlink == os.ModeSymlink {
wwlog.Debug("Found symlink %s", location)
destination, err := os.Readlink(location)
wwlog.Debug("Found symlink %s", walkPath)
target, err := os.Readlink(walkPath)
if err != nil {
wwlog.ErrorExc(err, "")
return fmt.Errorf("failed reading symlink: %w", err)
}
if util.IsFile(path.Join(outputDir, location)) {
if !util.IsFile(path.Join(outputDir, location+".wwbackup")) {
wwlog.Debug("Target exists, creating backup file")
err = os.Rename(path.Join(outputDir, location), path.Join(outputDir, location+".wwbackup"))
if util.IsFile(outputPath) {
backupPath := outputPath + ".wwbackup"
if !util.IsFile(backupPath) {
wwlog.Debug("Output file already exists: moving to backup file")
if err = os.Rename(outputPath, backupPath); err != nil {
return fmt.Errorf("failed renaming to backup file: %w", err)
}
} else {
wwlog.Debug("%s exists, keeping the backup file", path.Join(outputDir, location+".wwbackup"))
err = os.Remove(path.Join(outputDir, location))
}
if err != nil {
wwlog.ErrorExc(err, "")
wwlog.Debug("%s exists, keeping the backup file", backupPath)
if err = os.Remove(outputPath); err != nil {
return fmt.Errorf("failed removing existing file: %w", err)
}
}
}
err = os.Symlink(destination, path.Join(outputDir, location))
if err != nil {
wwlog.ErrorExc(err, "")
if err = os.Symlink(target, outputPath); err != nil {
return fmt.Errorf("failed creating symlink: %w", err)
}
wwlog.Debug("Created symlink file: %s", outputPath)
} else {
err := util.CopyFile(location, path.Join(outputDir, location))
if err == nil {
wwlog.Debug("Copied file into overlay: %s", location)
} else {
if err := util.CopyFile(walkPath, outputPath); err != nil {
return fmt.Errorf("could not copy file into overlay: %w", err)
}
wwlog.Debug("Copied overlay file: %s", outputPath)
}
return nil
})
if err != nil {
return fmt.Errorf("failed to build overlay working directory: %w", err)
return fmt.Errorf("failed to build overlay image directory: %w", err)
}
}

728
internal/pkg/overlay/overlay_test.go
View File

@@ -1,9 +1,11 @@
package overlay
import (
"fmt"
"io"
"os"
"path"
"runtime"
"sort"
"testing"
@@ -11,8 +13,9 @@ import (
warewulfconf "github.com/warewulf/warewulf/internal/pkg/config"
"github.com/warewulf/warewulf/internal/pkg/node"
"github.com/warewulf/warewulf/internal/pkg/testenv"
"github.com/warewulf/warewulf/internal/pkg/util"
"github.com/cavaliergopher/cpio"
)
func Test_OverlayMethods(t *testing.T) {
@@ -110,158 +113,310 @@ func Test_OverlayMethods(t *testing.T) {
}
}
var buildOverlayTests = []struct {
description string
nodeName string
context string
overlays []string
image string
contents []string
hasFiles bool
}{
{
description: "if no node, context, or overlays are specified then no overlay image is generated",
nodeName: "",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only node is specified then no overlay image is generated",
nodeName: "node1",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only context is specified then context named image is generated",
nodeName: "",
context: "system",
overlays: nil,
image: "__SYSTEM__.img",
contents: nil,
},
{
description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1"},
image: "o1.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1", "o2"},
image: "o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if a single node overlay is specified, then the overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1"},
image: "node1/o1.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1", "o2"},
image: "node1/o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if no node system overlays are specified, then context pointed overlay is generated",
nodeName: "node1",
context: "system",
overlays: nil,
image: "node1/__SYSTEM__.img",
contents: nil,
},
{
description: "if no node runtime overlays are specified, then context pointed overlay is generated",
nodeName: "node1",
context: "runtime",
overlays: nil,
image: "node1/__RUNTIME__.img",
contents: nil,
},
{
description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "system",
overlays: []string{"o1"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt"},
},
{
description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "system",
overlays: []string{"o1", "o2"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1", "o2"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt", "o2.txt"},
},
func Test_BuildOverlayIndir(t *testing.T) {
var tests = map[string]struct {
node node.Node
overlays []string
overlayFiles map[string]string
overlayDirs []string
overlaySymlinks map[string]string
outputDir string
outputFiles map[string]string
outputDirs []string
outputSymlinks map[string]string
}{
"empty": {
outputDir: "/image",
},
"empty directory": {
overlays: []string{"o1"},
overlayDirs: []string{"/var/lib/warewulf/overlays/o1/rootfs/testdir"},
outputDir: "/image",
outputDirs: []string{"testdir"},
},
"single flat file": {
overlays: []string{"o1"},
overlayFiles: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/testfile": "A test file from o1",
},
outputDir: "/image",
outputFiles: map[string]string{
"testfile": "A test file from o1",
},
},
"multiple overlays": {
overlays: []string{"o1", "o2"},
overlayFiles: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/t1.txt": "A test file from o1",
"/var/lib/warewulf/overlays/o2/rootfs/t2.txt": "A test file from o2",
},
outputDir: "/image",
outputFiles: map[string]string{
"t1.txt": "A test file from o1",
"t2.txt": "A test file from o2",
},
},
"template": {
node: node.Node{Profile: node.Profile{Comment: "A node comment"}},
overlays: []string{"o1"},
overlayFiles: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/t1.txt.ww": "{{ .Comment }}",
"/image/t1.txt": "Previous content",
},
outputDir: "/image",
outputFiles: map[string]string{
"t1.txt": "A node comment",
"t1.txt.wwbackup": "Previous content",
},
},
"multifile": {
overlays: []string{"o1"},
overlayFiles: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/file.txt.ww": `
{{- file "t1.txt" }}
T1
{{ file "t2.txt" }}
T2
{{ file "t3.txt" }}
T3
`,
},
outputDir: "/image",
outputFiles: map[string]string{
"t1.txt": "T1\n",
"t2.txt": "T2\n",
"t3.txt": "T3\n",
},
},
"symlink": {
overlays: []string{"o1"},
overlaySymlinks: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/test-link": "/test-target",
},
outputDir: "/image",
outputSymlinks: map[string]string{
"test-link": "/test-target",
},
},
"symlink from template": {
overlays: []string{"o1"},
overlayFiles: map[string]string{
"/var/lib/warewulf/overlays/o1/rootfs/test-link.ww": `
{{- softlink "/test-target" }}
`,
},
outputDir: "/image",
outputSymlinks: map[string]string{
"test-link": "/test-target",
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
env := testenv.New(t)
defer env.RemoveAll(t)
for fileName, content := range tt.overlayFiles {
env.WriteFile(t, fileName, content)
}
for _, dirName := range tt.overlayDirs {
env.MkdirAll(t, dirName)
}
for linkName, target := range tt.overlaySymlinks {
env.Symlink(t, target, linkName)
}
env.MkdirAll(t, tt.outputDir)
assert.NoError(t, BuildOverlayIndir(tt.node, []node.Node{tt.node}, tt.overlays, env.GetPath(tt.outputDir)))
dirFiles := tt.outputDirs
for outputFile, _ := range tt.outputFiles {
dirFiles = append(dirFiles, outputFile)
}
for outputFile, _ := range tt.outputSymlinks {
dirFiles = append(dirFiles, outputFile)
}
sort.Strings(dirFiles)
assert.Equal(t, dirFiles, env.ReadDir(t, tt.outputDir))
for fileName, content := range tt.outputFiles {
assert.Equal(t, content, env.ReadFile(t, path.Join(tt.outputDir, fileName)))
}
for _, dirName := range tt.outputDirs {
assert.True(t, util.IsDir(env.GetPath(path.Join(tt.outputDir, dirName))), fmt.Sprintf("%s is not a directory", dirName))
}
for linkName, expTarget := range tt.outputSymlinks {
target, err := os.Readlink(env.GetPath(path.Join(tt.outputDir, linkName)))
assert.NoError(t, err)
assert.Equal(t, expTarget, target)
}
})
}
}
func Test_BuildOverlay(t *testing.T) {
conf := warewulfconf.Get()
overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*")
assert.NoError(t, overlayDirErr)
defer os.RemoveAll(overlayDir)
conf.Paths.WWOverlaydir = overlayDir
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700))
{
_, err := os.Create(path.Join(overlayDir, "o1", "o1.txt"))
assert.NoError(t, err)
}
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700))
{
_, err := os.Create(path.Join(overlayDir, "o2", "o2.txt"))
assert.NoError(t, err)
var tests = []struct {
description string
nodeName string
context string
overlays []string
image string
contents []string
perms []int
hasFiles bool
}{
{
description: "if no node, context, or overlays are specified then no overlay image is generated",
nodeName: "",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only node is specified then no overlay image is generated",
nodeName: "node1",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only context is specified then context named image is generated",
nodeName: "",
context: "system",
overlays: nil,
image: "__SYSTEM__.img",
contents: nil,
},
{
description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1"},
image: "o1.img",
contents: []string{"o1.txt"},
perms: []int{0644},
},
{
description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1", "o2"},
image: "o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
perms: []int{0644, 0644},
},
{
description: "if a single node overlay is specified, then the overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1"},
image: "node1/o1.img",
contents: []string{"o1.txt"},
perms: []int{0644},
},
{
description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1", "o2"},
image: "node1/o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
perms: []int{0644, 0644},
},
{
description: "if no node system overlays are specified, then context pointed overlay is generated",
nodeName: "node1",
context: "system",
overlays: nil,
image: "node1/__SYSTEM__.img",
contents: nil,
},
{
description: "if no node runtime overlays are specified, then context pointed overlay is generated",
nodeName: "node1",
context: "runtime",
overlays: nil,
image: "node1/__RUNTIME__.img",
contents: nil,
},
{
description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "system",
overlays: []string{"o1"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt"},
perms: []int{0644},
},
{
description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt"},
perms: []int{0644},
},
{
description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "system",
overlays: []string{"o1", "o2"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt", "o2.txt"},
perms: []int{0644, 0644},
},
{
description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1", "o2"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt", "o2.txt"},
perms: []int{0644, 0644},
},
{
description: "validating altered permissions are retained",
nodeName: "node1",
context: "runtime",
overlays: []string{"o3"},
image: "node1/__RUNTIME__.img",
contents: []string{"subdir", "subdir/o3.txt"},
perms: []int{0700, 0600},
},
}
for _, tt := range buildOverlayTests {
env := testenv.New(t)
defer env.RemoveAll(t)
env.CreateFile(t, "var/lib/warewulf/overlays/o1/rootfs/o1.txt")
env.CreateFile(t, "var/lib/warewulf/overlays/o2/rootfs/o2.txt")
env.CreateFile(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww")
env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir", 0700)
env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww", 0600)
for _, tt := range tests {
nodeInfo := node.NewNode(tt.nodeName)
t.Run(tt.description, func(t *testing.T) {
provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*")
assert.NoError(t, provisionDirErr)
defer os.RemoveAll(provisionDir)
conf.Paths.WWProvisiondir = provisionDir
err := BuildOverlay(nodeInfo, tt.context, tt.overlays)
err := BuildOverlay(nodeInfo, []node.Node{nodeInfo}, tt.context, tt.overlays)
assert.NoError(t, err)
if tt.image != "" {
image := path.Join(provisionDir, "overlays", tt.image)
image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image))
assert.FileExists(t, image)
sort.Strings(tt.contents)
files, err := util.CpioFiles(image)
headers, err := readCpio(image)
assert.NoError(t, err)
sort.Strings(files)
assert.Equal(t, tt.contents, files)
assert.Equal(t, len(tt.contents), len(headers))
for i, file := range tt.contents {
assert.Equal(t, file, headers[file].Name)
if len(tt.perms) > i {
assert.Equal(t, tt.perms[i], int(headers[file].Mode.Perm()))
}
}
} else {
dirName := path.Join(provisionDir, "overlays", tt.nodeName)
dirName := env.GetPath(path.Join("srv/warewulf/overlays", tt.nodeName))
isEmpty := dirIsEmpty(t, dirName)
assert.True(t, isEmpty, "%v should be empty, but isn't", dirName)
}
@@ -269,83 +424,79 @@ func Test_BuildOverlay(t *testing.T) {
}
}
// Although these tests specify system and runtime overlays for the
// nodes, these overlays define the overlays that are defined in the
// configuration. BuildAllOverlays doesn't receive these as arguments,
// but builds all the overlays that are configured on the given node.
var buildAllOverlaysTests = []struct {
description string
nodes []string
systemOverlays [][]string
runtimeOverlays [][]string
createdOverlays []string
}{
{
description: "empty input creates no overlays",
nodes: nil,
systemOverlays: nil,
runtimeOverlays: nil,
createdOverlays: nil,
},
{
description: "a node with no overlays creates default system/runtime overlays",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: [][]string{{"o1"}},
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"},
},
{
description: "multiple nodes with no overlays creates default system/runtime overlays",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o2"}},
runtimeOverlays: [][]string{{"o1"}, {"o2"}},
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"},
},
{
description: "a system overlay for a node generates a system overlay for that node",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with different system overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"},
},
{
description: "two nodes with a single runtime overlay generates a runtime overlay for the first node",
nodes: []string{"node1"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz"},
},
{
description: "two nodes with different runtime overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"},
},
{
description: "a node with both a runtime and system overlay generates an image for each",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: [][]string{{"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with both runtime and system overlays generates each image for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: [][]string{{"o2"}, {"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"},
},
}
func Test_BuildAllOverlays(t *testing.T) {
var tests = []struct {
description string
nodes []string
systemOverlays [][]string
runtimeOverlays [][]string
createdOverlays []string
}{
{
description: "empty input creates no overlays",
nodes: nil,
systemOverlays: nil,
runtimeOverlays: nil,
createdOverlays: nil,
},
{
description: "a node with no overlays creates default system/runtime overlays",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: [][]string{{"o1"}},
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"},
},
{
description: "multiple nodes with no overlays creates default system/runtime overlays",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o2"}},
runtimeOverlays: [][]string{{"o1"}, {"o2"}},
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"},
},
{
description: "a system overlay for a node generates a system overlay for that node",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with different system overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"},
},
{
description: "two nodes with a single runtime overlay generates a runtime overlay for the first node",
nodes: []string{"node1"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz"},
},
{
description: "two nodes with different runtime overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"},
},
{
description: "a node with both a runtime and system overlay generates an image for each",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: [][]string{{"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with both runtime and system overlays generates each image for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: [][]string{{"o2"}, {"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"},
},
}
conf := warewulfconf.Get()
overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*")
assert.NoError(t, overlayDirErr)
@@ -354,7 +505,7 @@ func Test_BuildAllOverlays(t *testing.T) {
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700))
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700))
for _, tt := range buildAllOverlaysTests {
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*")
assert.NoError(t, provisionDirErr)
@@ -372,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) {
}
nodes = append(nodes, nodeInfo)
}
err := BuildAllOverlays(nodes)
err := BuildAllOverlays(nodes, nodes, runtime.NumCPU())
assert.NoError(t, err)
if tt.createdOverlays == nil {
dirName := path.Join(provisionDir, "overlays")
@@ -385,65 +536,65 @@ func Test_BuildAllOverlays(t *testing.T) {
}
}
var buildSpecificOverlaysTests = []struct {
description string
nodes []string
overlays []string
images []string
succeed bool
}{
{
description: "building no overlays for no nodes generates no error and no images",
nodes: nil,
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for a node generates no error and no images",
nodes: []string{"node1"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for two nodes generates no error and no images",
nodes: []string{"node1", "node2"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building an overlay for a node generates an overlay image in that node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1"},
images: []string{"node1/o1.img"},
succeed: true,
},
{
description: "building an overlay for two nodes generates an overlay image in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1"},
images: []string{"node1/o1.img", "node2/o1.img"},
succeed: true,
},
{
description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img"},
succeed: true,
},
{
description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"},
succeed: true,
},
}
func Test_BuildSpecificOverlays(t *testing.T) {
var tests = []struct {
description string
nodes []string
overlays []string
images []string
succeed bool
}{
{
description: "building no overlays for no nodes generates no error and no images",
nodes: nil,
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for a node generates no error and no images",
nodes: []string{"node1"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for two nodes generates no error and no images",
nodes: []string{"node1", "node2"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building an overlay for a node generates an overlay image in that node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1"},
images: []string{"node1/o1.img"},
succeed: true,
},
{
description: "building an overlay for two nodes generates an overlay image in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1"},
images: []string{"node1/o1.img", "node2/o1.img"},
succeed: true,
},
{
description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img"},
succeed: true,
},
{
description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"},
succeed: true,
},
}
conf := warewulfconf.Get()
overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*")
assert.NoError(t, overlayDirErr)
@@ -452,7 +603,7 @@ func Test_BuildSpecificOverlays(t *testing.T) {
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700))
assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700))
for _, tt := range buildSpecificOverlaysTests {
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*")
assert.NoError(t, provisionDirErr)
@@ -464,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) {
nodeInfo := node.NewNode(nodeName)
nodes = append(nodes, nodeInfo)
}
err := BuildSpecificOverlays(nodes, tt.overlays)
err := BuildSpecificOverlays(nodes, nodes, tt.overlays, runtime.NumCPU())
if !tt.succeed {
assert.Error(t, err)
} else {
@@ -493,3 +644,24 @@ func dirIsEmpty(t *testing.T, name string) bool {
t.Log(dirnames)
return false
}
func readCpio(name string) (headers map[string]*cpio.Header, err error) {
f, err := os.Open(name)
if err != nil {
return headers, err
}
defer f.Close()
reader := cpio.NewReader(f)
headers = make(map[string]*cpio.Header)
for {
header, err := reader.Next()
if err == io.EOF {
return headers, nil
}
if err != nil {
return headers, err
}
headers[header.Name] = header
}
}

34
internal/pkg/testenv/testenv.go
View File

@@ -112,6 +112,14 @@ func (env *TestEnv) MkdirAll(t *testing.T, dirName string) {
assert.NoError(t, err)
}
// Chmod changes the mode of fileName in the test environment, which must already exist.
//
// Asserts no errors occur.
func (env *TestEnv) Chmod(t *testing.T, fileName string, mode int) {
err := os.Chmod(env.GetPath(fileName), os.FileMode(mode))
assert.NoError(t, err)
}
// WriteFile writes content to fileName, creating any necessary
// intermediate directories relative to the test environment.
//
@@ -140,12 +148,22 @@ func (env *TestEnv) ImportFile(t *testing.T, fileName string, inputFileName stri
env.WriteFile(t, fileName, string(buffer))
}
// CreateFile creates an empty file to fileName, creating any necessary intermediate directories
// CreateFile creates an empty file at fileName, creating any necessary intermediate directories
// relative to the test environment.
func (env *TestEnv) CreateFile(t *testing.T, fileName string) {
env.WriteFile(t, fileName, "")
}
// Symlink creates a symlink at fileName to target, creating any necessary intermediate directories
// relative to the test environment.
func (env *TestEnv) Symlink(t *testing.T, target string, fileName string) {
dirName := filepath.Dir(fileName)
env.MkdirAll(t, dirName)
err := os.Symlink(target, env.GetPath(fileName))
assert.NoError(t, err)
}
// ReadFile returns the content of fileName as converted to a
// string.
//
@@ -156,6 +174,20 @@ func (env *TestEnv) ReadFile(t *testing.T, fileName string) string {
return string(buffer)
}
// ReadDir returns the content of dirName as converted to a
// slice of strings.
//
// Asserts no errors occur.
func (env *TestEnv) ReadDir(t *testing.T, dirName string) []string {
entries, err := os.ReadDir(env.GetPath(dirName))
assert.NoError(t, err)
var entryStrs []string
for _, entry := range entries {
entryStrs = append(entryStrs, entry.Name())
}
return entryStrs
}
// RemoveAll deletes the temporary directory, and all its contents,
// for the test environment.
//

31
internal/pkg/util/cpio.go
View File

@@ -1,31 +0,0 @@
package util
import (
"io"
"os"
"github.com/sassoftware/go-rpmutils/cpio"
)
/*
Opens cpio archive and returns the file list
*/
func CpioFiles(name string) (files []string, err error) {
f, err := os.Open(name)
if err != nil {
return files, err
}
defer f.Close()
reader := cpio.NewReader(f)
for {
header, err := reader.Next()
if err == io.EOF {
return files, nil
}
if err != nil {
return files, err
}
files = append(files, header.Filename())
}
}

136
internal/pkg/util/util.go
View File

@@ -18,10 +18,6 @@ import (
"github.com/warewulf/warewulf/internal/pkg/wwlog"
)
func BoolP(p *bool) bool {
return p != nil && *p
}
func FirstError(errs ...error) (err error) {
for _, e := range errs {
if err == nil {
@@ -136,37 +132,40 @@ func ValidString(pattern string, expr string) bool {
return false
}
// ******************************************************************************
func FindFiles(path string) []string {
var ret []string
wwlog.Debug("Changing directory to FindFiles path: %s", path)
err := os.Chdir(path)
if err != nil {
wwlog.Warn("Could not chdir() to: %s", path)
return ret
}
wwlog.Debug("Finding files in path: %s", path)
err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error {
err := filepath.Walk(path, func(location string, info os.FileInfo, err error) error {
if err != nil {
wwlog.Warn("Error walking path %s: %v", location, err)
return err
}
if location == "." {
// Get the relative path from the base directory
relPath, relErr := filepath.Rel(path, location)
if relErr != nil {
wwlog.Warn("Error computing relative path for %s: %v", location, relErr)
return relErr
}
if relPath == "." {
return nil
}
if IsDir(location) {
wwlog.Debug("FindFiles() found directory: %s", location)
ret = append(ret, location+"/")
if info.IsDir() {
wwlog.Debug("FindFiles() found directory: %s", relPath)
ret = append(ret, relPath+"/")
} else {
wwlog.Debug("FindFiles() found file: %s", location)
ret = append(ret, location)
wwlog.Debug("FindFiles() found file: %s", relPath)
ret = append(ret, relPath)
}
return nil
})
if err != nil {
wwlog.Warn("Error during file walk: %v", err)
return ret
}
@@ -184,82 +183,100 @@ func FindFilterFiles(
ignorePattern []string,
ignore_xdev bool) (ofiles []string, err error) {
wwlog.Debug("Finding files: %s include: %s ignore: %s", path, includePattern, ignorePattern)
// preprocess patterns to remove leading (and trailing) /, as we are handling relative paths
// Preprocess patterns to remove leading (and trailing) /, as we are handling relative paths
for i, pattern := range ignorePattern {
ignorePattern[i] = strings.Trim(pattern, "/")
}
cwd, err := os.Getwd()
// Convert the base path to an absolute path
absPath, err := filepath.Abs(path)
if err != nil {
return ofiles, err
return ofiles, fmt.Errorf("failed to resolve absolute path: %s: %w", path, err)
}
defer func() {
err = FirstError(err, os.Chdir(cwd))
}()
err = os.Chdir(path)
if err != nil {
return ofiles, fmt.Errorf("failed to change path: %s: %w", path, err)
}
// expand our include list as fspath.Match with /foo/* would catch /foo/baar but
// not /foo/baar/sibling
// Expand the include list
var globedInclude []string
for _, include := range includePattern {
globed, err := filepath.Glob(include)
globed, err := filepath.Glob(filepath.Join(absPath, include))
if err != nil {
return ofiles, err
return ofiles, fmt.Errorf("failed to glob pattern %s: %w", include, err)
}
globedInclude = append(globedInclude, globed...)
}
var dev uint64
if ignore_xdev {
wwlog.Debug("Ignoring cross-device (xdev) files")
pathStat, err := os.Stat(absPath)
if err != nil {
return ofiles, fmt.Errorf("failed to stat base path: %s: %w", absPath, err)
}
dev = pathStat.Sys().(*syscall.Stat_t).Dev
}
path_stat, err := os.Stat(".")
if err != nil {
return ofiles, err
}
dev := path_stat.Sys().(*syscall.Stat_t).Dev
for _, inc := range globedInclude {
wwlog.Debug("inc %s", inc)
wwlog.Debug("Processing include pattern: %s", inc)
stat, err := os.Lstat(inc)
if err != nil {
return ofiles, err
return ofiles, fmt.Errorf("failed to stat include: %s: %w", inc, err)
}
if stat.IsDir() {
// get the rest of dir
// Walk the directory
err = filepath.WalkDir(inc, func(location string, info fs.DirEntry, err error) error {
if err != nil {
return err
}
if location == "." {
relPath, relErr := filepath.Rel(absPath, location)
if relErr != nil {
wwlog.Warn("Error computing relative path for %s: %v", location, relErr)
return relErr
}
if relPath == "." {
return nil
}
fsInfo, err := info.Info()
if err != nil {
return err
}
if ignore_xdev && fsInfo.Sys().(*syscall.Stat_t).Dev != dev {
wwlog.Debug("Ignored (cross-device): %s", location)
return nil
}
for _, ignored_pat := range ignorePattern {
if ignored, _ := filepath.Match(ignored_pat, location); ignored {
wwlog.Debug("Ignored %s due to pattern %s", location, ignored_pat)
return filepath.SkipDir
for _, ignoredPattern := range ignorePattern {
if ignored, _ := filepath.Match(ignoredPattern, relPath); ignored {
wwlog.Debug("Ignored %s due to pattern %s", relPath, ignoredPattern)
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
}
ofiles = append(ofiles, location)
ofiles = append(ofiles, relPath)
return nil
})
if err != nil {
return ofiles, err
return ofiles, fmt.Errorf("error walking directory %s: %w", inc, err)
}
} else {
ofiles = append(ofiles, inc)
// Add the file directly
relPath, relErr := filepath.Rel(absPath, inc)
if relErr != nil {
wwlog.Warn("Error computing relative path for %s: %v", inc, relErr)
return ofiles, relErr
}
ofiles = append(ofiles, relPath)
}
}
return ofiles, err
return ofiles, nil
}
// ******************************************************************************
@@ -346,6 +363,7 @@ func AppendLines(fileName string, lines []string) error {
Create an archive using cpio
*/
func CpioCreate(
rootdir string,
ifiles []string,
ofile string,
format string,
@@ -354,7 +372,8 @@ func CpioCreate(
args := []string{
"--quiet",
"--create",
"-H", format,
"--directory", rootdir,
"--format", format,
"--file=" + ofile}
args = append(args, cpio_args...)
@@ -492,21 +511,9 @@ func BuildFsImage(
return fmt.Errorf("failed to create image directory for %s: %s: %w", name, imagePath, err)
}
wwlog.Debug("Created image directory for %s: %s", name, imagePath)
cwd, err := os.Getwd()
if err != nil {
return err
}
defer func() {
err = FirstError(err, os.Chdir(cwd))
}()
err = os.Chdir(rootfsPath)
if err != nil {
return fmt.Errorf("failed chdir to fs directory for %s: %s: %w", name, rootfsPath, err)
}
wwlog.Verbose("changed to: %s", rootfsPath)
files, err := FindFilterFiles(
".",
rootfsPath,
include,
ignore,
ignore_xdev)
@@ -515,6 +522,7 @@ func BuildFsImage(
}
err = CpioCreate(
rootfsPath,
files,
imagePath,
format,

113
internal/pkg/util/util_test.go
View File

@@ -1,53 +1,94 @@
package util
import (
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/warewulf/warewulf/internal/pkg/wwlog"
"github.com/warewulf/warewulf/internal/pkg/testenv"
)
func TryCreatePath(t *testing.T, elem ...string) {
err := os.MkdirAll(filepath.Join(elem...), os.ModePerm)
if err != nil {
t.Errorf("Failed creating dir: %v", err)
t.FailNow()
func Test_FindFiles(t *testing.T) {
var tests = map[string]struct {
createFiles []string
findFiles []string
}{
"no files": {
createFiles: []string{},
findFiles: nil,
},
"single file": {
createFiles: []string{"testfile"},
findFiles: []string{"testfile"},
},
"nested file": {
createFiles: []string{"testdir/testfile"},
findFiles: []string{"testdir/", "testdir/testfile"},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
env := testenv.New(t)
defer env.RemoveAll(t)
env.MkdirAll(t, "/test")
for _, file_ := range tt.createFiles {
env.CreateFile(t, filepath.Join("/test", file_))
}
files := FindFiles(env.GetPath("/test"))
assert.Equal(t, tt.findFiles, files)
})
}
}
func Test_FindFilterFiles(t *testing.T) {
wwlog.SetLogLevel(wwlog.DEBUG)
dir, err := os.MkdirTemp(os.TempDir(), "warewulf-test")
if err != nil {
t.Errorf("Failed creating tmpdir: %v", err)
t.FailNow()
}
defer os.RemoveAll(dir)
TryCreatePath(t, dir, "boot")
TryCreatePath(t, dir, "usr", "local")
TryCreatePath(t, dir, "usr", "bin")
TryCreatePath(t, dir, "usr", "usr", "local")
TryCreatePath(t, dir, "bin")
TryCreatePath(t, dir, "lib")
assert.NoError(t, os.Symlink("/path/to/target", filepath.Join(dir, "symlink")))
files, err := FindFilterFiles(dir, []string{"boot", "usr", "bin", "symlink"}, []string{"/b*/", "/usr/local"}, true)
if err != nil {
t.Errorf("FindFilerFiles failed: %v", err)
t.FailNow()
var tests = map[string]struct {
createFiles []string
include []string
exclude []string
findFiles []string
}{
"no files": {
createFiles: []string{},
include: []string{"*"},
findFiles: nil,
},
"single file": {
createFiles: []string{"testfile"},
include: []string{"*"},
findFiles: []string{"testfile"},
},
"nested file": {
createFiles: []string{"testdir/testfile"},
include: []string{"*"},
findFiles: []string{"testdir", "testdir/testfile"},
},
"multiple files": {
createFiles: []string{"test1/testfile", "test2/testfile"},
include: []string{"*"},
findFiles: []string{"test1", "test1/testfile", "test2", "test2/testfile"},
},
"excluded files": {
createFiles: []string{"test1/test1", "test1/test2", "test2/test1", "test2/test2"},
include: []string{"*"},
exclude: []string{"test1/*2", "test2"},
findFiles: []string{"test1", "test1/test1"},
},
}
expected := []string{"usr", "usr/bin", "usr/usr", "usr/usr/local", "symlink"}
sort.Strings(expected)
sort.Strings(files)
if !reflect.DeepEqual(files, expected) {
t.Errorf("expected %v, got %v", expected, files)
t.FailNow()
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
env := testenv.New(t)
defer env.RemoveAll(t)
env.MkdirAll(t, "/test")
for _, file_ := range tt.createFiles {
env.CreateFile(t, filepath.Join("/test", file_))
}
files, err := FindFilterFiles(env.GetPath("/test"), tt.include, tt.exclude, true)
assert.NoError(t, err)
assert.Equal(t, tt.findFiles, files)
})
}
}

19
internal/pkg/warewulfd/util.go
View File

@@ -45,9 +45,7 @@ func sendFile(
}
func getOverlayFile(n node.Node, context string, stage_overlays []string, autobuild bool) (stage_file string, err error) {
stage_file = overlay.OverlayImage(n.Id(), context, stage_overlays)
err = nil
build := !util.IsFile(stage_file)
wwlog.Verbose("stage file: %s", stage_file)
if !build && autobuild {
@@ -60,10 +58,23 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu
}
if build {
registry, err := node.New()
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
n.Id(), stage_overlays, stage_file, err)
return "", err
}
var allNodes []node.Node
allNodes, err = registry.FindAllNodes()
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
n.Id(), stage_overlays, stage_file, err)
return "", err
}
if len(stage_overlays) > 0 {
err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays)
err = overlay.BuildSpecificOverlays([]node.Node{n}, allNodes, stage_overlays, 1)
} else {
err = overlay.BuildAllOverlays([]node.Node{n})
err = overlay.BuildAllOverlays([]node.Node{n}, allNodes, 1)
}
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",

23
userdocs/contents/overlays.rst
View File

@@ -224,24 +224,13 @@ built. Specific overlays can be selected with ``-O`` flag. For
debugging purposes the templates can be written to a directory given
via the ``-o`` flag.
On clusters with large numbers of nodes a significant speedup can be achieved
by building overlays in parallel. Adding parallel overlay building to `wwctl`
is planned, see issue `#1087 <https://github.com/warewulf/warewulf/issues/1087>`_.
Until parallel overlay building is implemented, builds can be easily done in
parallel with Gnu `parallel` or `xargs`, for example:
Overlay images for multiple node are built in parallel. By default, each CPU in
the Warewulf server will build overlays independently. The number of workers
can be specified with the ``--workers`` option.
.. code-block:: console
# Gnu parallel
wwctl node list | awk '{print $1}' | grep -v "NODE " | parallel -j 12 \
wwctl overlay build -N {}
# xargs
wwctl node list | awk '{print $1}' | grep -v "NODE " | xargs -n 1 -P 12 \
wwctl overlay build -N
By default Warewulf will build/update and cache overlays as needed
(configurable in the ``warewulf.conf``).
Warewulf will attempt to build/update overlays as needed
(configurable in the ``warewulf.conf``); but not all cases are detected,
and manual overlay builds are often necessary.
Chmod
-----