From b0164c36b93c4c2ea60b57117c42ca1b7647bb22 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 19 Dec 2024 23:34:50 -0700 Subject: [PATCH 1/7] Remove Chdir from util.FindFiles Signed-off-by: Jonathon Anderson --- internal/pkg/config/buildconfig.go.in | 14 +++++------ internal/pkg/config/dhcp.go | 6 +---- internal/pkg/config/mounts.go | 8 ++---- internal/pkg/config/nfs.go | 6 ++--- internal/pkg/config/util.go | 5 ++++ internal/pkg/util/util.go | 35 +++++++++++++-------------- internal/pkg/util/util_test.go | 35 +++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 internal/pkg/config/util.go diff --git a/internal/pkg/config/buildconfig.go.in b/internal/pkg/config/buildconfig.go.in index 45dd5969..9b3a5859 100644 --- a/internal/pkg/config/buildconfig.go.in +++ b/internal/pkg/config/buildconfig.go.in @@ -2,8 +2,6 @@ package config import ( "path" - - "github.com/warewulf/warewulf/internal/pkg/util" ) var ConfigFile = "@SYSCONFDIR@/warewulf/warewulf.conf" @@ -36,7 +34,7 @@ type TFTPConf struct { } func (this TFTPConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } // WarewulfConf adds additional Warewulf-specific configuration to @@ -52,23 +50,23 @@ type WarewulfConf struct { } func (this WarewulfConf) Secure() bool { - return util.BoolP(this.SecureP) + return BoolP(this.SecureP) } func (this WarewulfConf) AutobuildOverlays() bool { - return util.BoolP(this.AutobuildOverlaysP) + return BoolP(this.AutobuildOverlaysP) } func (this WarewulfConf) EnableHostOverlay() bool { - return util.BoolP(this.EnableHostOverlayP) + return BoolP(this.EnableHostOverlayP) } func (this WarewulfConf) Syslog() bool { - return util.BoolP(this.SyslogP) + return BoolP(this.SyslogP) } func (this WarewulfConf) GrubBoot() bool { - return util.BoolP(this.GrubBootP) + return BoolP(this.GrubBootP) } func (paths BuildConfig) NodesConf() string { diff --git a/internal/pkg/config/dhcp.go b/internal/pkg/config/dhcp.go index 56c70986..7149594d 100644 --- a/internal/pkg/config/dhcp.go +++ b/internal/pkg/config/dhcp.go @@ -1,9 +1,5 @@ package config -import ( - "github.com/warewulf/warewulf/internal/pkg/util" -) - // DHCPConf represents the configuration for the DHCP service that // Warewulf will configure. type DHCPConf struct { @@ -15,5 +11,5 @@ type DHCPConf struct { } func (this DHCPConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } diff --git a/internal/pkg/config/mounts.go b/internal/pkg/config/mounts.go index f228a4e5..bc4cf920 100644 --- a/internal/pkg/config/mounts.go +++ b/internal/pkg/config/mounts.go @@ -1,9 +1,5 @@ package config -import ( - "github.com/warewulf/warewulf/internal/pkg/util" -) - // A MountEntry represents a bind mount that is applied to a container // during exec and shell. type MountEntry struct { @@ -15,9 +11,9 @@ type MountEntry struct { } func (this MountEntry) ReadOnly() bool { - return util.BoolP(this.ReadOnlyP) + return BoolP(this.ReadOnlyP) } func (this MountEntry) Copy() bool { - return util.BoolP(this.CopyP) + return BoolP(this.CopyP) } diff --git a/internal/pkg/config/nfs.go b/internal/pkg/config/nfs.go index ca6fc8b3..061eb199 100644 --- a/internal/pkg/config/nfs.go +++ b/internal/pkg/config/nfs.go @@ -2,8 +2,6 @@ package config import ( "github.com/creasty/defaults" - - "github.com/warewulf/warewulf/internal/pkg/util" ) // NFSConf represents the NFS configuration that will be used by @@ -16,7 +14,7 @@ type NFSConf struct { } func (this NFSConf) Enabled() bool { - return util.BoolP(this.EnabledP) + return BoolP(this.EnabledP) } // An NFSExportConf reprents a single NFS export / mount. @@ -28,7 +26,7 @@ type NFSExportConf struct { } func (this NFSExportConf) Mount() bool { - return util.BoolP(this.MountP) + return BoolP(this.MountP) } // Implements the Unmarshal interface for NFSConf to set default diff --git a/internal/pkg/config/util.go b/internal/pkg/config/util.go new file mode 100644 index 00000000..1198d46b --- /dev/null +++ b/internal/pkg/config/util.go @@ -0,0 +1,5 @@ +package config + +func BoolP(p *bool) bool { + return p != nil && *p +} diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index bf57fed4..a47037e3 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -18,10 +18,6 @@ import ( "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func BoolP(p *bool) bool { - return p != nil && *p -} - func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -136,37 +132,40 @@ func ValidString(pattern string, expr string) bool { return false } -// ****************************************************************************** func FindFiles(path string) []string { var ret []string - wwlog.Debug("Changing directory to FindFiles path: %s", path) - err := os.Chdir(path) - if err != nil { - wwlog.Warn("Could not chdir() to: %s", path) - return ret - } + wwlog.Debug("Finding files in path: %s", path) - err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + err := filepath.Walk(path, func(location string, info os.FileInfo, err error) error { if err != nil { + wwlog.Warn("Error walking path %s: %v", location, err) return err } - if location == "." { + // Get the relative path from the base directory + relPath, relErr := filepath.Rel(path, location) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", location, relErr) + return relErr + } + + if relPath == "." { return nil } - if IsDir(location) { - wwlog.Debug("FindFiles() found directory: %s", location) - ret = append(ret, location+"/") + if info.IsDir() { + wwlog.Debug("FindFiles() found directory: %s", relPath) + ret = append(ret, relPath+"/") } else { - wwlog.Debug("FindFiles() found file: %s", location) - ret = append(ret, location) + wwlog.Debug("FindFiles() found file: %s", relPath) + ret = append(ret, relPath) } return nil }) if err != nil { + wwlog.Warn("Error during file walk: %v", err) return ret } diff --git a/internal/pkg/util/util_test.go b/internal/pkg/util/util_test.go index e5087212..a8a4f4c1 100644 --- a/internal/pkg/util/util_test.go +++ b/internal/pkg/util/util_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) @@ -19,6 +20,40 @@ func TryCreatePath(t *testing.T, elem ...string) { } } +func Test_FindFiles(t *testing.T) { + var tests = map[string]struct { + createFiles []string + findFiles []string + }{ + "no files": { + createFiles: []string{}, + findFiles: nil, + }, + "single file": { + createFiles: []string{"testfile"}, + findFiles: []string{"testfile"}, + }, + "nested file": { + createFiles: []string{"testdir/testfile"}, + findFiles: []string{"testdir/", "testdir/testfile"}, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + env.MkdirAll(t, "/test") + for _, file_ := range tt.createFiles { + env.CreateFile(t, filepath.Join("/test", file_)) + } + + files := FindFiles(env.GetPath("/test")) + assert.Equal(t, tt.findFiles, files) + }) + } +} + func Test_FindFilterFiles(t *testing.T) { wwlog.SetLogLevel(wwlog.DEBUG) dir, err := os.MkdirTemp(os.TempDir(), "warewulf-test") From 9dd1ca0d536f43305f5923be3e65449f3fd0be41 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Fri, 20 Dec 2024 00:35:08 -0700 Subject: [PATCH 2/7] Removed Chdir from util.FindFilterFiles Signed-off-by: Jonathon Anderson --- internal/pkg/util/util.go | 82 ++++++++++++++++++++------------- internal/pkg/util/util_test.go | 84 ++++++++++++++++++---------------- 2 files changed, 95 insertions(+), 71 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index a47037e3..245c2adf 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -183,82 +183,100 @@ func FindFilterFiles( ignorePattern []string, ignore_xdev bool) (ofiles []string, err error) { wwlog.Debug("Finding files: %s include: %s ignore: %s", path, includePattern, ignorePattern) - // preprocess patterns to remove leading (and trailing) /, as we are handling relative paths + + // Preprocess patterns to remove leading (and trailing) /, as we are handling relative paths for i, pattern := range ignorePattern { ignorePattern[i] = strings.Trim(pattern, "/") } - cwd, err := os.Getwd() + + // Convert the base path to an absolute path + absPath, err := filepath.Abs(path) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to resolve absolute path: %s: %w", path, err) } - defer func() { - err = FirstError(err, os.Chdir(cwd)) - }() - err = os.Chdir(path) - if err != nil { - return ofiles, fmt.Errorf("failed to change path: %s: %w", path, err) - } - // expand our include list as fspath.Match with /foo/* would catch /foo/baar but - // not /foo/baar/sibling + + // Expand the include list var globedInclude []string for _, include := range includePattern { - globed, err := filepath.Glob(include) + globed, err := filepath.Glob(filepath.Join(absPath, include)) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to glob pattern %s: %w", include, err) } globedInclude = append(globedInclude, globed...) } + + var dev uint64 if ignore_xdev { wwlog.Debug("Ignoring cross-device (xdev) files") + pathStat, err := os.Stat(absPath) + if err != nil { + return ofiles, fmt.Errorf("failed to stat base path: %s: %w", absPath, err) + } + dev = pathStat.Sys().(*syscall.Stat_t).Dev } - path_stat, err := os.Stat(".") - if err != nil { - return ofiles, err - } - - dev := path_stat.Sys().(*syscall.Stat_t).Dev for _, inc := range globedInclude { - wwlog.Debug("inc %s", inc) + wwlog.Debug("Processing include pattern: %s", inc) stat, err := os.Lstat(inc) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("failed to stat include: %s: %w", inc, err) } + if stat.IsDir() { - // get the rest of dir + // Walk the directory err = filepath.WalkDir(inc, func(location string, info fs.DirEntry, err error) error { if err != nil { return err } - if location == "." { + + relPath, relErr := filepath.Rel(absPath, location) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", location, relErr) + return relErr + } + + if relPath == "." { return nil } + fsInfo, err := info.Info() if err != nil { return err } + if ignore_xdev && fsInfo.Sys().(*syscall.Stat_t).Dev != dev { wwlog.Debug("Ignored (cross-device): %s", location) return nil } - for _, ignored_pat := range ignorePattern { - if ignored, _ := filepath.Match(ignored_pat, location); ignored { - wwlog.Debug("Ignored %s due to pattern %s", location, ignored_pat) - return filepath.SkipDir + + for _, ignoredPattern := range ignorePattern { + if ignored, _ := filepath.Match(ignoredPattern, relPath); ignored { + wwlog.Debug("Ignored %s due to pattern %s", relPath, ignoredPattern) + if info.IsDir() { + return filepath.SkipDir + } + return nil } } - ofiles = append(ofiles, location) + + ofiles = append(ofiles, relPath) return nil }) if err != nil { - return ofiles, err + return ofiles, fmt.Errorf("error walking directory %s: %w", inc, err) } } else { - ofiles = append(ofiles, inc) + // Add the file directly + relPath, relErr := filepath.Rel(absPath, inc) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", inc, relErr) + return ofiles, relErr + } + ofiles = append(ofiles, relPath) } } - return ofiles, err + return ofiles, nil } // ****************************************************************************** diff --git a/internal/pkg/util/util_test.go b/internal/pkg/util/util_test.go index a8a4f4c1..b81710cc 100644 --- a/internal/pkg/util/util_test.go +++ b/internal/pkg/util/util_test.go @@ -1,25 +1,13 @@ package util import ( - "os" "path/filepath" - "reflect" - "sort" "testing" "github.com/stretchr/testify/assert" "github.com/warewulf/warewulf/internal/pkg/testenv" - "github.com/warewulf/warewulf/internal/pkg/wwlog" ) -func TryCreatePath(t *testing.T, elem ...string) { - err := os.MkdirAll(filepath.Join(elem...), os.ModePerm) - if err != nil { - t.Errorf("Failed creating dir: %v", err) - t.FailNow() - } -} - func Test_FindFiles(t *testing.T) { var tests = map[string]struct { createFiles []string @@ -55,34 +43,52 @@ func Test_FindFiles(t *testing.T) { } func Test_FindFilterFiles(t *testing.T) { - wwlog.SetLogLevel(wwlog.DEBUG) - dir, err := os.MkdirTemp(os.TempDir(), "warewulf-test") - if err != nil { - t.Errorf("Failed creating tmpdir: %v", err) - t.FailNow() - } - defer os.RemoveAll(dir) - TryCreatePath(t, dir, "boot") - TryCreatePath(t, dir, "usr", "local") - TryCreatePath(t, dir, "usr", "bin") - TryCreatePath(t, dir, "usr", "usr", "local") - TryCreatePath(t, dir, "bin") - TryCreatePath(t, dir, "lib") - - assert.NoError(t, os.Symlink("/path/to/target", filepath.Join(dir, "symlink"))) - - files, err := FindFilterFiles(dir, []string{"boot", "usr", "bin", "symlink"}, []string{"/b*/", "/usr/local"}, true) - - if err != nil { - t.Errorf("FindFilerFiles failed: %v", err) - t.FailNow() + var tests = map[string]struct { + createFiles []string + include []string + exclude []string + findFiles []string + }{ + "no files": { + createFiles: []string{}, + include: []string{"*"}, + findFiles: nil, + }, + "single file": { + createFiles: []string{"testfile"}, + include: []string{"*"}, + findFiles: []string{"testfile"}, + }, + "nested file": { + createFiles: []string{"testdir/testfile"}, + include: []string{"*"}, + findFiles: []string{"testdir", "testdir/testfile"}, + }, + "multiple files": { + createFiles: []string{"test1/testfile", "test2/testfile"}, + include: []string{"*"}, + findFiles: []string{"test1", "test1/testfile", "test2", "test2/testfile"}, + }, + "excluded files": { + createFiles: []string{"test1/test1", "test1/test2", "test2/test1", "test2/test2"}, + include: []string{"*"}, + exclude: []string{"test1/*2", "test2"}, + findFiles: []string{"test1", "test1/test1"}, + }, } - expected := []string{"usr", "usr/bin", "usr/usr", "usr/usr/local", "symlink"} - sort.Strings(expected) - sort.Strings(files) - if !reflect.DeepEqual(files, expected) { - t.Errorf("expected %v, got %v", expected, files) - t.FailNow() + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + env.MkdirAll(t, "/test") + for _, file_ := range tt.createFiles { + env.CreateFile(t, filepath.Join("/test", file_)) + } + + files, err := FindFilterFiles(env.GetPath("/test"), tt.include, tt.exclude, true) + assert.NoError(t, err) + assert.Equal(t, tt.findFiles, files) + }) } } From e3680438a67febd107bcc331011d2b633b2fe52b Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Fri, 20 Dec 2024 12:06:03 -0700 Subject: [PATCH 3/7] Remove Chdir from overlay.BuildOverlayIndir Signed-off-by: Jonathon Anderson --- internal/pkg/overlay/overlay.go | 196 ++++---- internal/pkg/overlay/overlay_test.go | 639 ++++++++++++++++----------- internal/pkg/testenv/testenv.go | 26 +- internal/pkg/warewulfd/util.go | 2 - 4 files changed, 513 insertions(+), 350 deletions(-) diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 5628d004..3c65fbcb 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -14,7 +14,6 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/pkg/errors" "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" @@ -23,7 +22,7 @@ import ( ) var ( - ErrDoesNotExist = errors.New("overlay does not exist") + ErrDoesNotExist = fmt.Errorf("overlay does not exist") ) // Overlay represents an overlay directory path. @@ -231,155 +230,160 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err return err } -/* -Build the given overlays for a node in the given directory. If the given does not -exists it will be created. -*/ +var regFile *regexp.Regexp +var regLink *regexp.Regexp + +func init() { + regFile = regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`) + regLink = regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`) +} + +// Build the given overlays for a node in the given directory. func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return nil } if !util.IsDir(outputDir) { - return errors.Errorf("output must a be a directory: %s", outputDir) + return fmt.Errorf("output must a be a directory: %s", outputDir) } if !util.ValidString(strings.Join(overlayNames, ""), "^[a-zA-Z0-9-._:]+$") { - return errors.Errorf("overlay names contains illegal characters: %v", overlayNames) + return fmt.Errorf("overlay names contains illegal characters: %v", overlayNames) } // Temporarily set umask to 0000, so directories in the overlay retain permissions defer syscall.Umask(syscall.Umask(0)) - wwlog.Verbose("Processing node/overlay: %s/%s", nodeData.Id(), strings.Join(overlayNames, "-")) + wwlog.Verbose("Processing node/overlays: %s/%s", nodeData.Id(), strings.Join(overlayNames, ",")) for _, overlayName := range overlayNames { wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir) - overlaySourceDir := GetOverlay(overlayName).Rootfs() - wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir) - err := os.Chdir(overlaySourceDir) - if err != nil { - return fmt.Errorf("directory: %s name: %s err: %w", overlaySourceDir, overlayName, ErrDoesNotExist) + overlayRootfs := GetOverlay(overlayName).Rootfs() + if !util.IsDir(overlayRootfs) { + return fmt.Errorf("overlay %s: %w", overlayName, ErrDoesNotExist) } - wwlog.Verbose("Walking the overlay structure: %s", overlaySourceDir) - err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + wwlog.Debug("Walking the overlay structure: %s", overlayRootfs) + err := filepath.Walk(overlayRootfs, func(walkPath string, info os.FileInfo, err error) error { if err != nil { - return fmt.Errorf("error for %s: %w", location, err) + return fmt.Errorf("error for %s: %w", walkPath, err) } + wwlog.Debug("Found overlay file: %s", walkPath) - wwlog.Debug("Found overlay file: %s", location) + relPath, relErr := filepath.Rel(overlayRootfs, walkPath) + if relErr != nil { + wwlog.Warn("Error computing relative path for %s: %v", walkPath, relErr) + return relErr + } + outputPath := path.Join(outputDir, relPath) if info.IsDir() { - wwlog.Debug("Found directory: %s", location) + wwlog.Debug("Found directory: %s", walkPath) - err = os.MkdirAll(path.Join(outputDir, location), info.Mode()) - if err != nil { + if err = os.MkdirAll(outputPath, info.Mode()); err != nil { return fmt.Errorf("could not create directory within overlay: %w", err) } - err = util.CopyUIDGID(location, path.Join(outputDir, location)) - if err != nil { + if err = util.CopyUIDGID(walkPath, outputPath); err != nil { return fmt.Errorf("failed setting permissions on overlay directory: %w", err) } - wwlog.Debug("Created directory in overlay: %s", location) + wwlog.Debug("Created directory in overlay: %s", outputPath) - } else if filepath.Ext(location) == ".ww" { + } else if filepath.Ext(walkPath) == ".ww" { + originalOutputPath := outputPath + outputPath := strings.TrimSuffix(outputPath, ".ww") tstruct, err := InitStruct(overlayName, nodeData) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } - tstruct.BuildSource = path.Join(overlaySourceDir, location) - wwlog.Verbose("Evaluating overlay template file: %s", location) - destFile := strings.TrimSuffix(location, ".ww") + tstruct.BuildSource = walkPath + wwlog.Verbose("Evaluating overlay template file: %s", walkPath) - buffer, backupFile, writeFile, err := RenderTemplateFile(location, tstruct) + buffer, backupFile, writeFile, err := RenderTemplateFile(walkPath, tstruct) if err != nil { - return fmt.Errorf("failed to render template %s: %w", location, err) + return fmt.Errorf("failed to render template %s: %w", walkPath, err) } - if writeFile { - destFileName := destFile - var fileBuffer bytes.Buffer - // search for magic file name comment - fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) - fileScanner.Split(ScanLines) - regFile := regexp.MustCompile(`.*{{\s*/\*\s*file\s*["'](.*)["']\s*\*/\s*}}.*`) - regLink := regexp.MustCompile(`.*{{\s*/\*\s*softlink\s*["'](.*)["']\s*\*/\s*}}.*`) - foundFileComment := false - for fileScanner.Scan() { - line := fileScanner.Text() - filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1) - softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1) - if len(softlinkFromTemplate) != 0 { - wwlog.Debug("Creating soft link %s -> %s", destFileName, softlinkFromTemplate[0][1]) - return os.Symlink(softlinkFromTemplate[0][1], path.Join(outputDir, destFileName)) - } else if len(filenameFromTemplate) != 0 { - wwlog.Debug("Writing file %s", filenameFromTemplate[0][1]) - if foundFileComment { - err = CarefulWriteBuffer(path.Join(outputDir, destFileName), - fileBuffer, backupFile, info.Mode()) - if err != nil { - return fmt.Errorf("could not write file from template: %w", err) - } - err = util.CopyUIDGID(location, path.Join(outputDir, destFileName)) - if err != nil { - return fmt.Errorf("failed setting permissions on template output file: %w", err) - } - fileBuffer.Reset() + if !writeFile { + return nil + } + var fileBuffer bytes.Buffer + // search for magic file name comment + fileScanner := bufio.NewScanner(bytes.NewReader(buffer.Bytes())) + fileScanner.Split(ScanLines) + foundFileComment := false + for fileScanner.Scan() { + line := fileScanner.Text() + filenameFromTemplate := regFile.FindAllStringSubmatch(line, -1) + softlinkFromTemplate := regLink.FindAllStringSubmatch(line, -1) + if len(softlinkFromTemplate) != 0 { + wwlog.Debug("Creating soft link %s -> %s", outputPath, softlinkFromTemplate[0][1]) + return os.Symlink(softlinkFromTemplate[0][1], outputPath) + } else if len(filenameFromTemplate) != 0 { + wwlog.Debug("Writing file %s", filenameFromTemplate[0][1]) + if foundFileComment { + err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode()) + if err != nil { + return fmt.Errorf("could not write file from template: %w", err) } - destFileName = path.Join(path.Dir(destFile), filenameFromTemplate[0][1]) - foundFileComment = true - } else { - _, _ = fileBuffer.WriteString(line) + err = util.CopyUIDGID(walkPath, outputPath) + if err != nil { + return fmt.Errorf("failed setting permissions on template output file: %w", err) + } + fileBuffer.Reset() + } + outputPath = path.Join(path.Dir(originalOutputPath), filenameFromTemplate[0][1]) + foundFileComment = true + } else { + if _, err = fileBuffer.WriteString(line); err != nil { + return fmt.Errorf("could not write to template buffer: %w", err) } } - err = CarefulWriteBuffer(path.Join(outputDir, destFileName), fileBuffer, backupFile, info.Mode()) - if err != nil { - return fmt.Errorf("could not write file from template: %w", err) - } - err = util.CopyUIDGID(location, path.Join(outputDir, destFileName)) - if err != nil { - return fmt.Errorf("failed setting permissions on template output file: %w", err) - } - - wwlog.Debug("Wrote template file into overlay: %s", destFile) - - // } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b { - // wwlog.Debug("Ignoring WW template file: %s", location) } + err = CarefulWriteBuffer(outputPath, fileBuffer, backupFile, info.Mode()) + if err != nil { + return fmt.Errorf("could not write file from template: %w", err) + } + err = util.CopyUIDGID(walkPath, outputPath) + if err != nil { + return fmt.Errorf("failed setting permissions on template output file: %w", err) + } + wwlog.Debug("Wrote template file into overlay: %s", outputPath) + } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { - wwlog.Debug("Found symlink %s", location) - destination, err := os.Readlink(location) + wwlog.Debug("Found symlink %s", walkPath) + target, err := os.Readlink(walkPath) if err != nil { - wwlog.ErrorExc(err, "") + return fmt.Errorf("failed reading symlink: %w", err) } - if util.IsFile(path.Join(outputDir, location)) { - if !util.IsFile(path.Join(outputDir, location+".wwbackup")) { - wwlog.Debug("Target exists, creating backup file") - err = os.Rename(path.Join(outputDir, location), path.Join(outputDir, location+".wwbackup")) + if util.IsFile(outputPath) { + backupPath := outputPath + ".wwbackup" + if !util.IsFile(backupPath) { + wwlog.Debug("Output file already exists: moving to backup file") + if err = os.Rename(outputPath, backupPath); err != nil { + return fmt.Errorf("failed renaming to backup file: %w", err) + } } else { - wwlog.Debug("%s exists, keeping the backup file", path.Join(outputDir, location+".wwbackup")) - err = os.Remove(path.Join(outputDir, location)) - } - if err != nil { - wwlog.ErrorExc(err, "") + wwlog.Debug("%s exists, keeping the backup file", backupPath) + if err = os.Remove(outputPath); err != nil { + return fmt.Errorf("failed removing existing file: %w", err) + } } } - err = os.Symlink(destination, path.Join(outputDir, location)) - if err != nil { - wwlog.ErrorExc(err, "") + if err = os.Symlink(target, outputPath); err != nil { + return fmt.Errorf("failed creating symlink: %w", err) } + wwlog.Debug("Created symlink file: %s", outputPath) } else { - err := util.CopyFile(location, path.Join(outputDir, location)) - if err == nil { - wwlog.Debug("Copied file into overlay: %s", location) - } else { + if err := util.CopyFile(walkPath, outputPath); err != nil { return fmt.Errorf("could not copy file into overlay: %w", err) } + wwlog.Debug("Copied overlay file: %s", outputPath) } return nil }) + if err != nil { - return fmt.Errorf("failed to build overlay working directory: %w", err) + return fmt.Errorf("failed to build overlay image directory: %w", err) } } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index a409041d..52aa8725 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -1,6 +1,7 @@ package overlay import ( + "fmt" "io" "os" "path" @@ -11,7 +12,6 @@ import ( warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/testenv" - "github.com/warewulf/warewulf/internal/pkg/util" ) @@ -110,122 +110,263 @@ func Test_OverlayMethods(t *testing.T) { } } -var buildOverlayTests = []struct { - description string - nodeName string - context string - overlays []string - image string - contents []string - hasFiles bool -}{ - { - description: "if no node, context, or overlays are specified then no overlay image is generated", - nodeName: "", - context: "", - overlays: nil, - image: "", - contents: nil, - }, - { - description: "if only node is specified then no overlay image is generated", - nodeName: "node1", - context: "", - overlays: nil, - image: "", - contents: nil, - }, - { - description: "if only context is specified then context named image is generated", - nodeName: "", - context: "system", - overlays: nil, - image: "__SYSTEM__.img", - contents: nil, - }, - { - description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory", - nodeName: "", - context: "", - overlays: []string{"o1"}, - image: "o1.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory", - nodeName: "", - context: "", - overlays: []string{"o1", "o2"}, - image: "o1-o2.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if a single node overlay is specified, then the overlay is built in a node overlay directory", - nodeName: "node1", - context: "", - overlays: []string{"o1"}, - image: "node1/o1.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory", - nodeName: "node1", - context: "", - overlays: []string{"o1", "o2"}, - image: "node1/o1-o2.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if no node system overlays are specified, then context pointed overlay is generated", - nodeName: "node1", - context: "system", - overlays: nil, - image: "node1/__SYSTEM__.img", - contents: nil, - }, - { - description: "if no node runtime overlays are specified, then context pointed overlay is generated", - nodeName: "node1", - context: "runtime", - overlays: nil, - image: "node1/__RUNTIME__.img", - contents: nil, - }, - { - description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory", - nodeName: "node1", - context: "system", - overlays: []string{"o1"}, - image: "node1/__SYSTEM__.img", - contents: []string{"o1.txt"}, - }, - { - description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory", - nodeName: "node1", - context: "runtime", - overlays: []string{"o1"}, - image: "node1/__RUNTIME__.img", - contents: []string{"o1.txt"}, - }, - { - description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays", - nodeName: "node1", - context: "system", - overlays: []string{"o1", "o2"}, - image: "node1/__SYSTEM__.img", - contents: []string{"o1.txt", "o2.txt"}, - }, - { - description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays", - nodeName: "node1", - context: "runtime", - overlays: []string{"o1", "o2"}, - image: "node1/__RUNTIME__.img", - contents: []string{"o1.txt", "o2.txt"}, - }, +func Test_BuildOverlayIndir(t *testing.T) { + var tests = map[string]struct { + node node.Node + overlays []string + overlayFiles map[string]string + overlayDirs []string + overlaySymlinks map[string]string + outputDir string + outputFiles map[string]string + outputDirs []string + outputSymlinks map[string]string + }{ + "empty": { + outputDir: "/image", + }, + "empty directory": { + overlays: []string{"o1"}, + overlayDirs: []string{"/var/lib/warewulf/overlays/o1/rootfs/testdir"}, + outputDir: "/image", + outputDirs: []string{"testdir"}, + }, + "single flat file": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/testfile": "A test file from o1", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "testfile": "A test file from o1", + }, + }, + "multiple overlays": { + overlays: []string{"o1", "o2"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/t1.txt": "A test file from o1", + "/var/lib/warewulf/overlays/o2/rootfs/t2.txt": "A test file from o2", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "A test file from o1", + "t2.txt": "A test file from o2", + }, + }, + "template": { + node: node.Node{Profile: node.Profile{Comment: "A node comment"}}, + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/t1.txt.ww": "{{ .Comment }}", + "/image/t1.txt": "Previous content", + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "A node comment", + "t1.txt.wwbackup": "Previous content", + }, + }, + "multifile": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/file.txt.ww": ` +{{- file "t1.txt" }} +T1 +{{ file "t2.txt" }} +T2 +{{ file "t3.txt" }} +T3 +`, + }, + outputDir: "/image", + outputFiles: map[string]string{ + "t1.txt": "T1\n", + "t2.txt": "T2\n", + "t3.txt": "T3\n", + }, + }, + "symlink": { + overlays: []string{"o1"}, + overlaySymlinks: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/test-link": "/test-target", + }, + outputDir: "/image", + outputSymlinks: map[string]string{ + "test-link": "/test-target", + }, + }, + "symlink from template": { + overlays: []string{"o1"}, + overlayFiles: map[string]string{ + "/var/lib/warewulf/overlays/o1/rootfs/test-link.ww": ` +{{- softlink "/test-target" }} +`, + }, + outputDir: "/image", + outputSymlinks: map[string]string{ + "test-link": "/test-target", + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + env := testenv.New(t) + defer env.RemoveAll(t) + + for fileName, content := range tt.overlayFiles { + env.WriteFile(t, fileName, content) + } + for _, dirName := range tt.overlayDirs { + env.MkdirAll(t, dirName) + } + for linkName, target := range tt.overlaySymlinks { + env.Symlink(t, target, linkName) + } + + env.MkdirAll(t, tt.outputDir) + + assert.NoError(t, BuildOverlayIndir(tt.node, tt.overlays, env.GetPath(tt.outputDir))) + dirFiles := tt.outputDirs + for outputFile, _ := range tt.outputFiles { + dirFiles = append(dirFiles, outputFile) + } + for outputFile, _ := range tt.outputSymlinks { + dirFiles = append(dirFiles, outputFile) + } + sort.Strings(dirFiles) + assert.Equal(t, dirFiles, env.ReadDir(t, tt.outputDir)) + for fileName, content := range tt.outputFiles { + assert.Equal(t, content, env.ReadFile(t, path.Join(tt.outputDir, fileName))) + } + for _, dirName := range tt.outputDirs { + assert.True(t, util.IsDir(env.GetPath(path.Join(tt.outputDir, dirName))), fmt.Sprintf("%s is not a directory", dirName)) + } + for linkName, expTarget := range tt.outputSymlinks { + target, err := os.Readlink(env.GetPath(path.Join(tt.outputDir, linkName))) + assert.NoError(t, err) + assert.Equal(t, expTarget, target) + } + }) + } } func Test_BuildOverlay(t *testing.T) { + var tests = []struct { + description string + nodeName string + context string + overlays []string + image string + contents []string + hasFiles bool + }{ + { + description: "if no node, context, or overlays are specified then no overlay image is generated", + nodeName: "", + context: "", + overlays: nil, + image: "", + contents: nil, + }, + { + description: "if only node is specified then no overlay image is generated", + nodeName: "node1", + context: "", + overlays: nil, + image: "", + contents: nil, + }, + { + description: "if only context is specified then context named image is generated", + nodeName: "", + context: "system", + overlays: nil, + image: "__SYSTEM__.img", + contents: nil, + }, + { + description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory", + nodeName: "", + context: "", + overlays: []string{"o1"}, + image: "o1.img", + contents: []string{"o1.txt"}, + }, + { + description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory", + nodeName: "", + context: "", + overlays: []string{"o1", "o2"}, + image: "o1-o2.img", + contents: []string{"o1.txt", "o2.txt"}, + }, + { + description: "if a single node overlay is specified, then the overlay is built in a node overlay directory", + nodeName: "node1", + context: "", + overlays: []string{"o1"}, + image: "node1/o1.img", + contents: []string{"o1.txt"}, + }, + { + description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory", + nodeName: "node1", + context: "", + overlays: []string{"o1", "o2"}, + image: "node1/o1-o2.img", + contents: []string{"o1.txt", "o2.txt"}, + }, + { + description: "if no node system overlays are specified, then context pointed overlay is generated", + nodeName: "node1", + context: "system", + overlays: nil, + image: "node1/__SYSTEM__.img", + contents: nil, + }, + { + description: "if no node runtime overlays are specified, then context pointed overlay is generated", + nodeName: "node1", + context: "runtime", + overlays: nil, + image: "node1/__RUNTIME__.img", + contents: nil, + }, + { + description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory", + nodeName: "node1", + context: "system", + overlays: []string{"o1"}, + image: "node1/__SYSTEM__.img", + contents: []string{"o1.txt"}, + }, + { + description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory", + nodeName: "node1", + context: "runtime", + overlays: []string{"o1"}, + image: "node1/__RUNTIME__.img", + contents: []string{"o1.txt"}, + }, + { + description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays", + nodeName: "node1", + context: "system", + overlays: []string{"o1", "o2"}, + image: "node1/__SYSTEM__.img", + contents: []string{"o1.txt", "o2.txt"}, + }, + { + description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays", + nodeName: "node1", + context: "runtime", + overlays: []string{"o1", "o2"}, + image: "node1/__RUNTIME__.img", + contents: []string{"o1.txt", "o2.txt"}, + }, + } + conf := warewulfconf.Get() overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") assert.NoError(t, overlayDirErr) @@ -242,7 +383,7 @@ func Test_BuildOverlay(t *testing.T) { assert.NoError(t, err) } - for _, tt := range buildOverlayTests { + for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") @@ -269,83 +410,79 @@ func Test_BuildOverlay(t *testing.T) { } } -// Although these tests specify system and runtime overlays for the -// nodes, these overlays define the overlays that are defined in the -// configuration. BuildAllOverlays doesn't receive these as arguments, -// but builds all the overlays that are configured on the given node. -var buildAllOverlaysTests = []struct { - description string - nodes []string - systemOverlays [][]string - runtimeOverlays [][]string - createdOverlays []string -}{ - { - description: "empty input creates no overlays", - nodes: nil, - systemOverlays: nil, - runtimeOverlays: nil, - createdOverlays: nil, - }, - { - description: "a node with no overlays creates default system/runtime overlays", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: [][]string{{"o1"}}, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"}, - }, - { - description: "multiple nodes with no overlays creates default system/runtime overlays", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o2"}}, - runtimeOverlays: [][]string{{"o1"}, {"o2"}}, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"}, - }, - { - description: "a system overlay for a node generates a system overlay for that node", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: nil, - createdOverlays: []string{"node1/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with different system overlays generates a system overlay for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - runtimeOverlays: nil, - createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with a single runtime overlay generates a runtime overlay for the first node", - nodes: []string{"node1"}, - systemOverlays: nil, - runtimeOverlays: [][]string{{"o1"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz"}, - }, - { - description: "two nodes with different runtime overlays generates a system overlay for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: nil, - runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"}, - }, - { - description: "a node with both a runtime and system overlay generates an image for each", - nodes: []string{"node1"}, - systemOverlays: [][]string{{"o1"}}, - runtimeOverlays: [][]string{{"o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"}, - }, - { - description: "two nodes with both runtime and system overlays generates each image for each node", - nodes: []string{"node1", "node2"}, - systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, - runtimeOverlays: [][]string{{"o2"}, {"o2"}}, - createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"}, - }, -} - func Test_BuildAllOverlays(t *testing.T) { + var tests = []struct { + description string + nodes []string + systemOverlays [][]string + runtimeOverlays [][]string + createdOverlays []string + }{ + { + description: "empty input creates no overlays", + nodes: nil, + systemOverlays: nil, + runtimeOverlays: nil, + createdOverlays: nil, + }, + { + description: "a node with no overlays creates default system/runtime overlays", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: [][]string{{"o1"}}, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz"}, + }, + { + description: "multiple nodes with no overlays creates default system/runtime overlays", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o2"}}, + runtimeOverlays: [][]string{{"o1"}, {"o2"}}, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node1/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz"}, + }, + { + description: "a system overlay for a node generates a system overlay for that node", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: nil, + createdOverlays: []string{"node1/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with different system overlays generates a system overlay for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + runtimeOverlays: nil, + createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with a single runtime overlay generates a runtime overlay for the first node", + nodes: []string{"node1"}, + systemOverlays: nil, + runtimeOverlays: [][]string{{"o1"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz"}, + }, + { + description: "two nodes with different runtime overlays generates a system overlay for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: nil, + runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"}, + }, + { + description: "a node with both a runtime and system overlay generates an image for each", + nodes: []string{"node1"}, + systemOverlays: [][]string{{"o1"}}, + runtimeOverlays: [][]string{{"o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"}, + }, + { + description: "two nodes with both runtime and system overlays generates each image for each node", + nodes: []string{"node1", "node2"}, + systemOverlays: [][]string{{"o1"}, {"o1", "o2"}}, + runtimeOverlays: [][]string{{"o2"}, {"o2"}}, + createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"}, + }, + } + conf := warewulfconf.Get() overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") assert.NoError(t, overlayDirErr) @@ -354,7 +491,7 @@ func Test_BuildAllOverlays(t *testing.T) { assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - for _, tt := range buildAllOverlaysTests { + for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") assert.NoError(t, provisionDirErr) @@ -385,65 +522,65 @@ func Test_BuildAllOverlays(t *testing.T) { } } -var buildSpecificOverlaysTests = []struct { - description string - nodes []string - overlays []string - images []string - succeed bool -}{ - { - description: "building no overlays for no nodes generates no error and no images", - nodes: nil, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building no overlays for a node generates no error and no images", - nodes: []string{"node1"}, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building no overlays for two nodes generates no error and no images", - nodes: []string{"node1", "node2"}, - overlays: nil, - images: nil, - succeed: true, - }, - { - description: "building an overlay for a node generates an overlay image in that node's overlay directory", - nodes: []string{"node1"}, - overlays: []string{"o1"}, - images: []string{"node1/o1.img"}, - succeed: true, - }, - { - description: "building an overlay for two nodes generates an overlay image in each node's overlay directory", - nodes: []string{"node1", "node2"}, - overlays: []string{"o1"}, - images: []string{"node1/o1.img", "node2/o1.img"}, - succeed: true, - }, - { - description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory", - nodes: []string{"node1"}, - overlays: []string{"o1", "o2"}, - images: []string{"node1/o1.img", "node1/o2.img"}, - succeed: true, - }, - { - description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory", - nodes: []string{"node1", "node2"}, - overlays: []string{"o1", "o2"}, - images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"}, - succeed: true, - }, -} - func Test_BuildSpecificOverlays(t *testing.T) { + var tests = []struct { + description string + nodes []string + overlays []string + images []string + succeed bool + }{ + { + description: "building no overlays for no nodes generates no error and no images", + nodes: nil, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building no overlays for a node generates no error and no images", + nodes: []string{"node1"}, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building no overlays for two nodes generates no error and no images", + nodes: []string{"node1", "node2"}, + overlays: nil, + images: nil, + succeed: true, + }, + { + description: "building an overlay for a node generates an overlay image in that node's overlay directory", + nodes: []string{"node1"}, + overlays: []string{"o1"}, + images: []string{"node1/o1.img"}, + succeed: true, + }, + { + description: "building an overlay for two nodes generates an overlay image in each node's overlay directory", + nodes: []string{"node1", "node2"}, + overlays: []string{"o1"}, + images: []string{"node1/o1.img", "node2/o1.img"}, + succeed: true, + }, + { + description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory", + nodes: []string{"node1"}, + overlays: []string{"o1", "o2"}, + images: []string{"node1/o1.img", "node1/o2.img"}, + succeed: true, + }, + { + description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory", + nodes: []string{"node1", "node2"}, + overlays: []string{"o1", "o2"}, + images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"}, + succeed: true, + }, + } + conf := warewulfconf.Get() overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") assert.NoError(t, overlayDirErr) @@ -452,7 +589,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - for _, tt := range buildSpecificOverlaysTests { + for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") assert.NoError(t, provisionDirErr) diff --git a/internal/pkg/testenv/testenv.go b/internal/pkg/testenv/testenv.go index 115688e9..30405d24 100644 --- a/internal/pkg/testenv/testenv.go +++ b/internal/pkg/testenv/testenv.go @@ -140,12 +140,22 @@ func (env *TestEnv) ImportFile(t *testing.T, fileName string, inputFileName stri env.WriteFile(t, fileName, string(buffer)) } -// CreateFile creates an empty file to fileName, creating any necessary intermediate directories +// CreateFile creates an empty file at fileName, creating any necessary intermediate directories // relative to the test environment. func (env *TestEnv) CreateFile(t *testing.T, fileName string) { env.WriteFile(t, fileName, "") } +// Symlink creates a symlink at fileName to target, creating any necessary intermediate directories +// relative to the test environment. +func (env *TestEnv) Symlink(t *testing.T, target string, fileName string) { + dirName := filepath.Dir(fileName) + env.MkdirAll(t, dirName) + + err := os.Symlink(target, env.GetPath(fileName)) + assert.NoError(t, err) +} + // ReadFile returns the content of fileName as converted to a // string. // @@ -156,6 +166,20 @@ func (env *TestEnv) ReadFile(t *testing.T, fileName string) string { return string(buffer) } +// ReadDir returns the content of dirName as converted to a +// slice of strings. +// +// Asserts no errors occur. +func (env *TestEnv) ReadDir(t *testing.T, dirName string) []string { + entries, err := os.ReadDir(env.GetPath(dirName)) + assert.NoError(t, err) + var entryStrs []string + for _, entry := range entries { + entryStrs = append(entryStrs, entry.Name()) + } + return entryStrs +} + // RemoveAll deletes the temporary directory, and all its contents, // for the test environment. // diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index a40c1781..973cc6a6 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -45,9 +45,7 @@ func sendFile( } func getOverlayFile(n node.Node, context string, stage_overlays []string, autobuild bool) (stage_file string, err error) { - stage_file = overlay.OverlayImage(n.Id(), context, stage_overlays) - err = nil build := !util.IsFile(stage_file) wwlog.Verbose("stage file: %s", stage_file) if !build && autobuild { From 1affdc18838fdf2e1f9e7c6354043dd89a180de1 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Fri, 20 Dec 2024 12:11:33 -0700 Subject: [PATCH 4/7] Remove Chdir from util.BuildFsImage Signed-off-by: Jonathon Anderson --- internal/pkg/util/util.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 245c2adf..e037c343 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -363,6 +363,7 @@ func AppendLines(fileName string, lines []string) error { Create an archive using cpio */ func CpioCreate( + rootdir string, ifiles []string, ofile string, format string, @@ -371,7 +372,8 @@ func CpioCreate( args := []string{ "--quiet", "--create", - "-H", format, + "--directory", rootdir, + "--format", format, "--file=" + ofile} args = append(args, cpio_args...) @@ -509,21 +511,9 @@ func BuildFsImage( return fmt.Errorf("failed to create image directory for %s: %s: %w", name, imagePath, err) } wwlog.Debug("Created image directory for %s: %s", name, imagePath) - cwd, err := os.Getwd() - if err != nil { - return err - } - defer func() { - err = FirstError(err, os.Chdir(cwd)) - }() - err = os.Chdir(rootfsPath) - if err != nil { - return fmt.Errorf("failed chdir to fs directory for %s: %s: %w", name, rootfsPath, err) - } - wwlog.Verbose("changed to: %s", rootfsPath) files, err := FindFilterFiles( - ".", + rootfsPath, include, ignore, ignore_xdev) @@ -532,6 +522,7 @@ func BuildFsImage( } err = CpioCreate( + rootfsPath, files, imagePath, format, From 53e5805fbec4ef4469ac08769e59e8eb26daee5b Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Mon, 23 Dec 2024 09:47:28 -0700 Subject: [PATCH 5/7] Remove os.Umask from overlay.BuildOverlayIndir Adds tests to ensure that the overlay image contains the correct permissions. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 - go.mod | 2 +- go.sum | 4 +- internal/pkg/overlay/overlay.go | 4 -- internal/pkg/overlay/overlay_test.go | 84 +++++++++++++++++++--------- internal/pkg/testenv/testenv.go | 8 +++ internal/pkg/util/cpio.go | 31 ---------- 7 files changed, 70 insertions(+), 64 deletions(-) delete mode 100644 internal/pkg/util/cpio.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fe6191..3d45b57e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Bump github.com/spf13/cobra from 1.7.0 to 1.8.0 #1166 - Bump github.com/fatih/color from 1.15.0 to 1.17.0 #1224 - Bump github.com/coreos/ignition/v2 from 2.15.0 to 2.19.0 #1239 -- Bump github.com/sassoftware/go-rpmutils from 0.2.0 to 0.4.0 #1217 - Bump github.com/spf13/cobra from 1.8.0 to 1.8.1 #1481 - Bump google.golang.org/protobuf from 1.34.1 to 1.35.1 #1480 - Bump golang.org/x/term from 0.20.0 to 0.25.0 #1476 diff --git a/go.mod b/go.mod index d30a0489..a4f400fd 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.0 require ( dario.cat/mergo v1.0.1 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/cavaliergopher/cpio v1.0.1 github.com/cheynewallace/tabby v1.1.1 github.com/containers/image/v5 v5.32.2 github.com/containers/storage v1.55.2 @@ -21,7 +22,6 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/umoci v0.4.7 github.com/pkg/errors v0.9.1 - github.com/sassoftware/go-rpmutils v0.4.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/talos-systems/go-smbios v0.1.1 diff --git a/go.sum b/go.sum index 8a26eaae..0f46fa29 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= +github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -310,8 +312,6 @@ github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZP github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg= -github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI= github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 3c65fbcb..4ae8515f 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -10,7 +10,6 @@ import ( "path/filepath" "regexp" "strings" - "syscall" "text/template" "github.com/Masterminds/sprig/v3" @@ -251,9 +250,6 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri return fmt.Errorf("overlay names contains illegal characters: %v", overlayNames) } - // Temporarily set umask to 0000, so directories in the overlay retain permissions - defer syscall.Umask(syscall.Umask(0)) - wwlog.Verbose("Processing node/overlays: %s/%s", nodeData.Id(), strings.Join(overlayNames, ",")) for _, overlayName := range overlayNames { wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeData.Id(), outputDir) diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 52aa8725..673d6e68 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -13,6 +13,8 @@ import ( "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/util" + + "github.com/cavaliergopher/cpio" ) func Test_OverlayMethods(t *testing.T) { @@ -259,6 +261,7 @@ func Test_BuildOverlay(t *testing.T) { overlays []string image string contents []string + perms []int hasFiles bool }{ { @@ -292,6 +295,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1"}, image: "o1.img", contents: []string{"o1.txt"}, + perms: []int{0644}, }, { description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory", @@ -300,6 +304,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1", "o2"}, image: "o1-o2.img", contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, }, { description: "if a single node overlay is specified, then the overlay is built in a node overlay directory", @@ -308,6 +313,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1"}, image: "node1/o1.img", contents: []string{"o1.txt"}, + perms: []int{0644}, }, { description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory", @@ -316,6 +322,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1", "o2"}, image: "node1/o1-o2.img", contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, }, { description: "if no node system overlays are specified, then context pointed overlay is generated", @@ -340,6 +347,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1"}, image: "node1/__SYSTEM__.img", contents: []string{"o1.txt"}, + perms: []int{0644}, }, { description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory", @@ -348,6 +356,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1"}, image: "node1/__RUNTIME__.img", contents: []string{"o1.txt"}, + perms: []int{0644}, }, { description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays", @@ -356,6 +365,7 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1", "o2"}, image: "node1/__SYSTEM__.img", contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, }, { description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays", @@ -364,45 +374,48 @@ func Test_BuildOverlay(t *testing.T) { overlays: []string{"o1", "o2"}, image: "node1/__RUNTIME__.img", contents: []string{"o1.txt", "o2.txt"}, + perms: []int{0644, 0644}, + }, + { + description: "validating altered permissions are retained", + nodeName: "node1", + context: "runtime", + overlays: []string{"o3"}, + image: "node1/__RUNTIME__.img", + contents: []string{"subdir", "subdir/o3.txt"}, + perms: []int{0700, 0600}, }, } - conf := warewulfconf.Get() - overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*") - assert.NoError(t, overlayDirErr) - defer os.RemoveAll(overlayDir) - conf.Paths.WWOverlaydir = overlayDir - assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o1"), 0700)) - { - _, err := os.Create(path.Join(overlayDir, "o1", "o1.txt")) - assert.NoError(t, err) - } - assert.NoError(t, os.Mkdir(path.Join(overlayDir, "o2"), 0700)) - { - _, err := os.Create(path.Join(overlayDir, "o2", "o2.txt")) - assert.NoError(t, err) - } + env := testenv.New(t) + defer env.RemoveAll(t) + + env.CreateFile(t, "var/lib/warewulf/overlays/o1/rootfs/o1.txt") + env.CreateFile(t, "var/lib/warewulf/overlays/o2/rootfs/o2.txt") + env.CreateFile(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww") + env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir", 0700) + env.Chmod(t, "var/lib/warewulf/overlays/o3/rootfs/subdir/o3.txt.ww", 0600) for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { - provisionDir, provisionDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*") - assert.NoError(t, provisionDirErr) - defer os.RemoveAll(provisionDir) - conf.Paths.WWProvisiondir = provisionDir - err := BuildOverlay(nodeInfo, tt.context, tt.overlays) assert.NoError(t, err) if tt.image != "" { - image := path.Join(provisionDir, "overlays", tt.image) + image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image)) assert.FileExists(t, image) sort.Strings(tt.contents) - files, err := util.CpioFiles(image) + headers, err := readCpio(image) assert.NoError(t, err) - sort.Strings(files) - assert.Equal(t, tt.contents, files) + assert.Equal(t, len(tt.contents), len(headers)) + for i, file := range tt.contents { + assert.Equal(t, file, headers[file].Name) + if len(tt.perms) > i { + assert.Equal(t, tt.perms[i], int(headers[file].Mode.Perm())) + } + } } else { - dirName := path.Join(provisionDir, "overlays", tt.nodeName) + dirName := env.GetPath(path.Join("srv/warewulf/overlays", tt.nodeName)) isEmpty := dirIsEmpty(t, dirName) assert.True(t, isEmpty, "%v should be empty, but isn't", dirName) } @@ -630,3 +643,24 @@ func dirIsEmpty(t *testing.T, name string) bool { t.Log(dirnames) return false } + +func readCpio(name string) (headers map[string]*cpio.Header, err error) { + f, err := os.Open(name) + if err != nil { + return headers, err + } + defer f.Close() + + reader := cpio.NewReader(f) + headers = make(map[string]*cpio.Header) + for { + header, err := reader.Next() + if err == io.EOF { + return headers, nil + } + if err != nil { + return headers, err + } + headers[header.Name] = header + } +} diff --git a/internal/pkg/testenv/testenv.go b/internal/pkg/testenv/testenv.go index 30405d24..92642822 100644 --- a/internal/pkg/testenv/testenv.go +++ b/internal/pkg/testenv/testenv.go @@ -112,6 +112,14 @@ func (env *TestEnv) MkdirAll(t *testing.T, dirName string) { assert.NoError(t, err) } +// Chmod changes the mode of fileName in the test environment, which must already exist. +// +// Asserts no errors occur. +func (env *TestEnv) Chmod(t *testing.T, fileName string, mode int) { + err := os.Chmod(env.GetPath(fileName), os.FileMode(mode)) + assert.NoError(t, err) +} + // WriteFile writes content to fileName, creating any necessary // intermediate directories relative to the test environment. // diff --git a/internal/pkg/util/cpio.go b/internal/pkg/util/cpio.go deleted file mode 100644 index f8709c3d..00000000 --- a/internal/pkg/util/cpio.go +++ /dev/null @@ -1,31 +0,0 @@ -package util - -import ( - "io" - "os" - - "github.com/sassoftware/go-rpmutils/cpio" -) - -/* -Opens cpio archive and returns the file list -*/ -func CpioFiles(name string) (files []string, err error) { - f, err := os.Open(name) - if err != nil { - return files, err - } - defer f.Close() - - reader := cpio.NewReader(f) - for { - header, err := reader.Next() - if err == io.EOF { - return files, nil - } - if err != nil { - return files, err - } - files = append(files, header.Filename()) - } -} From a59403ab3b75aebba9631bce76fc9962f8a28cfa Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 19 Dec 2024 04:29:34 -0700 Subject: [PATCH 6/7] Parallel overlay builds - Closes #1018 Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + internal/app/wwctl/overlay/build/main.go | 4 +- internal/app/wwctl/overlay/build/root.go | 4 +- internal/app/wwctl/overlay/imprt/main.go | 2 +- internal/app/wwctl/overlay/imprt/root.go | 4 ++ internal/pkg/overlay/overlay.go | 86 +++++++++++++++++------- internal/pkg/overlay/overlay_test.go | 5 +- internal/pkg/warewulfd/util.go | 4 +- userdocs/contents/overlays.rst | 23 ++----- 9 files changed, 83 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d45b57e..e40f1419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added note to booting userdoc for removing machine-id. #1609 - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 +- Parallelized overlay build. #1018 ### Removed diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 5fb89b62..21ce42c9 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -66,9 +66,9 @@ func CobraRunE(cmd *cobra.Command, args []string) error { defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames) + err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db) + err = overlay.BuildAllOverlays(db, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index e24ad6b3..74055c6d 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -2,6 +2,7 @@ package build import ( "log" + "runtime" "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/node" @@ -31,6 +32,7 @@ var ( } OverlayNames []string OverlayDir string + Workers int ) func init() { @@ -44,7 +46,7 @@ func init() { } baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to the given directory. An overlay must also be ge given to use this option.`) - + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 8a890bb1..55d3a208 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}) + return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/imprt/root.go b/internal/app/wwctl/overlay/imprt/root.go index 6c7840d3..38359676 100644 --- a/internal/app/wwctl/overlay/imprt/root.go +++ b/internal/app/wwctl/overlay/imprt/root.go @@ -1,6 +1,8 @@ package imprt import ( + "runtime" + "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/overlay" ) @@ -24,11 +26,13 @@ var ( } NoOverlayUpdate bool CreateDirs bool + Workers int ) func init() { baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories") + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 4ae8515f..37724604 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "text/template" "github.com/Masterminds/sprig/v3" @@ -100,40 +101,76 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -/* -Build all overlays for a node -*/ -func BuildAllOverlays(nodes []node.Node) error { +func BuildAllOverlays(nodes []node.Node, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)*2) + + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) + if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil { + errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) + } + + wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) + if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil { + errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) + } + } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) - sysOverlays := n.SystemOverlay - wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(sysOverlays, ", ")) - err := BuildOverlay(n, "system", sysOverlays) - if err != nil { - return fmt.Errorf("could not build system overlays %v for node %s: %w", sysOverlays, n.Id(), err) - } - runOverlays := n.RuntimeOverlay - wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(runOverlays, ", ")) - err = BuildOverlay(n, "runtime", runOverlays) - if err != nil { - return fmt.Errorf("could not build runtime overlays %v for node %s: %w", runOverlays, n.Id(), err) - } + wg.Wait() + close(errChan) + for err := range errChan { + return err } return nil } -// TODO: Add an Overlay Delete for both sourcedir and image +func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)) -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string) error { - for _, n := range nodes { - wwlog.Info("Building overlay for %s: %v", n, overlayNames) - for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) - if err != nil { - return fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building overlay for %s: %v", n, overlayNames) + for _, overlayName := range overlayNames { + err := BuildOverlay(n, "", []string{overlayName}) + if err != nil { + errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + } } } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } + for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) + + wg.Wait() + close(errChan) + + for err := range errChan { + return err } return nil } @@ -177,7 +214,6 @@ func FindOverlays() (overlayList []string, err error) { overlayList = append(overlayList, file.Name()) } } - return overlayList, nil } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 673d6e68..5e198c4f 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path" + "runtime" "sort" "testing" @@ -522,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes) + err := BuildAllOverlays(nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -614,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays) + err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 973cc6a6..a98c1b47 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -59,9 +59,9 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu if build { if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays) + err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}) + err = overlay.BuildAllOverlays([]node.Node{n}, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", diff --git a/userdocs/contents/overlays.rst b/userdocs/contents/overlays.rst index a71439be..0327dc9c 100644 --- a/userdocs/contents/overlays.rst +++ b/userdocs/contents/overlays.rst @@ -224,24 +224,13 @@ built. Specific overlays can be selected with ``-O`` flag. For debugging purposes the templates can be written to a directory given via the ``-o`` flag. -On clusters with large numbers of nodes a significant speedup can be achieved -by building overlays in parallel. Adding parallel overlay building to `wwctl` -is planned, see issue `#1087 `_. -Until parallel overlay building is implemented, builds can be easily done in -parallel with Gnu `parallel` or `xargs`, for example: +Overlay images for multiple node are built in parallel. By default, each CPU in +the Warewulf server will build overlays independently. The number of workers +can be specified with the ``--workers`` option. -.. code-block:: console - - # Gnu parallel - wwctl node list | awk '{print $1}' | grep -v "NODE " | parallel -j 12 \ - wwctl overlay build -N {} - - # xargs - wwctl node list | awk '{print $1}' | grep -v "NODE " | xargs -n 1 -P 12 \ - wwctl overlay build -N - -By default Warewulf will build/update and cache overlays as needed -(configurable in the ``warewulf.conf``). +Warewulf will attempt to build/update overlays as needed +(configurable in the ``warewulf.conf``); but not all cases are detected, +and manual overlay builds are often necessary. Chmod ----- From c244adc7f69c64c9485b2a24440d11d46c87035d Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Mon, 23 Dec 2024 20:52:06 -0700 Subject: [PATCH 7/7] Performance improvements for overlay build Rendering overlay templates requires fetching and populating / merging all other nodes, because overlay templates have access to all cluster node info. This was previously being done for each template render, but is not being done once for each render operation and re-used for each template render. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + .../app/wwctl/container/exec/child/main.go | 12 ++++---- internal/app/wwctl/overlay/build/main.go | 20 +++++++------ internal/app/wwctl/overlay/imprt/main.go | 2 +- internal/app/wwctl/overlay/show/main.go | 7 ++++- internal/pkg/configure/hostfile.go | 12 +++++++- internal/pkg/overlay/datastructure.go | 17 ++--------- internal/pkg/overlay/overlay.go | 29 ++++++++++++------- internal/pkg/overlay/overlay_test.go | 8 ++--- internal/pkg/warewulfd/util.go | 17 +++++++++-- 10 files changed, 77 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40f1419..f088b583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 - Parallelized overlay build. #1018 +- Parallelized and optimized overlay build. #1018 ### Removed diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 2542f3c2..52c461b5 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -97,17 +97,17 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { return fmt.Errorf("could not open node configuration: %s", err) } - nodes, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } - nodes = node.FilterNodeListByName(nodes, []string{nodename}) - if len(nodes) != 1 { + filteredNodes := node.FilterNodeListByName(allNodes, []string{nodename}) + if len(filteredNodes) != 1 { return fmt.Errorf("no single node idendified with %s", nodename) } - overlays := nodes[0].SystemOverlay - overlays = append(overlays, nodes[0].RuntimeOverlay...) - err = overlay.BuildOverlayIndir(nodes[0], overlays, path.Join(runDir, "nodeoverlay")) + overlays := filteredNodes[0].SystemOverlay + overlays = append(overlays, filteredNodes[0].RuntimeOverlay...) + err = overlay.BuildOverlayIndir(filteredNodes[0], allNodes, overlays, path.Join(runDir, "nodeoverlay")) if err != nil { return fmt.Errorf("could not build overlay: %s", err) } diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 21ce42c9..2e987de7 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -18,18 +18,21 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("could not open node configuration: %s", err) } - db, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } + var filteredNodes []node.Node if len(args) > 0 { args = hostlist.Expand(args) - db = node.FilterNodeListByName(db, args) + filteredNodes = node.FilterNodeListByName(allNodes, args) - if len(db) < len(args) { + if len(filteredNodes) < len(args) { return errors.New("failed to find nodes") } + } else { + filteredNodes = allNodes } // NOTE: this is to keep backward compatible @@ -49,26 +52,25 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if len(args) > 0 { - if len(db) != 1 { + if len(filteredNodes) != 1 { return errors.New("must specify one node to build overlay") } - for _, node := range db { - return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) + for _, node := range filteredNodes { + return overlay.BuildOverlayIndir(node, allNodes, OverlayNames, OverlayDir) } } else { return errors.New("must specify a node to build overlay") } - } oldMask := syscall.Umask(007) defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers) + err = overlay.BuildSpecificOverlays(filteredNodes, allNodes, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db, Workers) + err = overlay.BuildAllOverlays(filteredNodes, allNodes, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 55d3a208..f53144f8 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers) + return overlay.BuildSpecificOverlays(updateNodes, nodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/show/main.go b/internal/app/wwctl/overlay/show/main.go index 97f7a3e4..533322e7 100644 --- a/internal/app/wwctl/overlay/show/main.go +++ b/internal/app/wwctl/overlay/show/main.go @@ -57,7 +57,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodeConf = node.NewNode(hostName) nodeConf.ClusterName = hostName } - tstruct, err := overlay.InitStruct(overlayName, nodeConf) + var allNodes []node.Node + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + tstruct, err := overlay.InitStruct(overlayName, nodeConf, allNodes) if err != nil { return err } diff --git a/internal/pkg/configure/hostfile.go b/internal/pkg/configure/hostfile.go index 3046eb41..e8bf1759 100644 --- a/internal/pkg/configure/hostfile.go +++ b/internal/pkg/configure/hostfile.go @@ -20,8 +20,18 @@ func Hostfile() (err error) { return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay") } + var allNodes []node.Node + if nodeDB, err := node.New(); err != nil { + return err + } else { + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + } + hostname, _ := os.Hostname() - tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname)) + tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname), allNodes) if err != nil { return err } diff --git a/internal/pkg/overlay/datastructure.go b/internal/pkg/overlay/datastructure.go index 67c61d5f..e818717f 100644 --- a/internal/pkg/overlay/datastructure.go +++ b/internal/pkg/overlay/datastructure.go @@ -46,16 +46,12 @@ type TemplateStruct struct { /* Initialize an TemplateStruct with the given node.NodeInfo */ -func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) { +func InitStruct(overlayName string, nodeData node.Node, allNodes []node.Node) (TemplateStruct, error) { var tstruct TemplateStruct tstruct.Overlay = overlayName hostname, _ := os.Hostname() tstruct.BuildHost = hostname controller := warewulfconf.Get() - nodeDB, err := node.New() - if err != nil { - return tstruct, err - } tstruct.ThisNode = &nodeData if tstruct.ThisNode.Kernel == nil { tstruct.ThisNode.Kernel = new(node.KernelConf) @@ -75,10 +71,6 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) tstruct.Ipaddr6 = controller.Ipaddr6 tstruct.Netmask = controller.Netmask tstruct.Network = controller.Network - allNodes, err := nodeDB.FindAllNodes() - if err != nil { - return tstruct, err - } // init some convenience vars tstruct.Id = nodeData.Id() tstruct.Hostname = nodeData.Id() @@ -92,14 +84,11 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) var buf bytes.Buffer enc := gob.NewEncoder(&buf) dec := gob.NewDecoder(&buf) - err = enc.Encode(nodeData) - if err != nil { + if err := enc.Encode(nodeData); err != nil { return tstruct, err } - err = dec.Decode(&tstruct) - if err != nil { + if err := dec.Decode(&tstruct); err != nil { return tstruct, err } return tstruct, nil - } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 37724604..e0eba792 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -101,7 +101,7 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -func BuildAllOverlays(nodes []node.Node, workerCount int) error { +func BuildAllOverlays(nodes []node.Node, allNodes []node.Node, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)*2) @@ -109,12 +109,12 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { worker := func() { for n := range nodeChan { wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) - if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "system", n.SystemOverlay); err != nil { errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) } wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) - if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "runtime", n.RuntimeOverlay); err != nil { errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) } } @@ -139,7 +139,7 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { return nil } -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error { +func BuildSpecificOverlays(nodes []node.Node, allNodes []node.Node, overlayNames []string, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)) @@ -148,7 +148,7 @@ func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount for n := range nodeChan { wwlog.Info("Building overlay for %s: %v", n, overlayNames) for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) + err := BuildOverlay(n, allNodes, "", []string{overlayName}) if err != nil { errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) } @@ -190,7 +190,16 @@ func BuildHostOverlay() error { if !(stats.Mode() == os.FileMode(0750|os.ModeDir) || stats.Mode() == os.FileMode(0700|os.ModeDir)) { wwlog.SecWarn("Permissions of host overlay dir %s are %s (750 is considered as secure)", hostdir, stats.Mode()) } - return BuildOverlayIndir(hostData, []string{"host"}, "/") + registry, err := node.New() + if err != nil { + return err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + return err + } + return BuildOverlayIndir(hostData, allNodes, []string{"host"}, "/") } /* @@ -220,7 +229,7 @@ func FindOverlays() (overlayList []string, err error) { /* Build the given overlays for a node and create a Image for them */ -func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) error { +func BuildOverlay(nodeConf node.Node, allNodes []node.Node, context string, overlayNames []string) error { if len(overlayNames) == 0 && context == "" { return nil } @@ -245,7 +254,7 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err wwlog.Debug("Created temporary directory for %s: %s", name, buildDir) - err = BuildOverlayIndir(nodeConf, overlayNames, buildDir) + err = BuildOverlayIndir(nodeConf, allNodes, overlayNames, buildDir) if err != nil { return fmt.Errorf("failed to generate files for %s: %w", name, err) } @@ -274,7 +283,7 @@ func init() { } // Build the given overlays for a node in the given directory. -func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error { +func BuildOverlayIndir(nodeData node.Node, allNodes []node.Node, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return nil } @@ -323,7 +332,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri } else if filepath.Ext(walkPath) == ".ww" { originalOutputPath := outputPath outputPath := strings.TrimSuffix(outputPath, ".ww") - tstruct, err := InitStruct(overlayName, nodeData) + tstruct, err := InitStruct(overlayName, nodeData, allNodes) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 5e198c4f..af6a09af 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -229,7 +229,7 @@ T3 env.MkdirAll(t, tt.outputDir) - assert.NoError(t, BuildOverlayIndir(tt.node, tt.overlays, env.GetPath(tt.outputDir))) + assert.NoError(t, BuildOverlayIndir(tt.node, []node.Node{tt.node}, tt.overlays, env.GetPath(tt.outputDir))) dirFiles := tt.outputDirs for outputFile, _ := range tt.outputFiles { dirFiles = append(dirFiles, outputFile) @@ -400,7 +400,7 @@ func Test_BuildOverlay(t *testing.T) { for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { - err := BuildOverlay(nodeInfo, tt.context, tt.overlays) + err := BuildOverlay(nodeInfo, []node.Node{nodeInfo}, tt.context, tt.overlays) assert.NoError(t, err) if tt.image != "" { image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image)) @@ -523,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes, runtime.NumCPU()) + err := BuildAllOverlays(nodes, nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -615,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU()) + err := BuildSpecificOverlays(nodes, nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index a98c1b47..919ee05e 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -58,10 +58,23 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu } if build { + registry, err := node.New() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1) + err = overlay.BuildSpecificOverlays([]node.Node{n}, allNodes, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}, 1) + err = overlay.BuildAllOverlays([]node.Node{n}, allNodes, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",