diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d45b57e..e40f1419 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added note to booting userdoc for removing machine-id. #1609 - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 +- Parallelized overlay build. #1018 ### Removed diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 5fb89b62..21ce42c9 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -66,9 +66,9 @@ func CobraRunE(cmd *cobra.Command, args []string) error { defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames) + err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db) + err = overlay.BuildAllOverlays(db, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index e24ad6b3..74055c6d 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -2,6 +2,7 @@ package build import ( "log" + "runtime" "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/node" @@ -31,6 +32,7 @@ var ( } OverlayNames []string OverlayDir string + Workers int ) func init() { @@ -44,7 +46,7 @@ func init() { } baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to the given directory. An overlay must also be ge given to use this option.`) - + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 8a890bb1..55d3a208 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}) + return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/imprt/root.go b/internal/app/wwctl/overlay/imprt/root.go index 6c7840d3..38359676 100644 --- a/internal/app/wwctl/overlay/imprt/root.go +++ b/internal/app/wwctl/overlay/imprt/root.go @@ -1,6 +1,8 @@ package imprt import ( + "runtime" + "github.com/spf13/cobra" "github.com/warewulf/warewulf/internal/pkg/overlay" ) @@ -24,11 +26,13 @@ var ( } NoOverlayUpdate bool CreateDirs bool + Workers int ) func init() { baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories") + baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 4ae8515f..37724604 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "text/template" "github.com/Masterminds/sprig/v3" @@ -100,40 +101,76 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -/* -Build all overlays for a node -*/ -func BuildAllOverlays(nodes []node.Node) error { +func BuildAllOverlays(nodes []node.Node, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)*2) + + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) + if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil { + errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) + } + + wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) + if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil { + errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) + } + } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) - sysOverlays := n.SystemOverlay - wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(sysOverlays, ", ")) - err := BuildOverlay(n, "system", sysOverlays) - if err != nil { - return fmt.Errorf("could not build system overlays %v for node %s: %w", sysOverlays, n.Id(), err) - } - runOverlays := n.RuntimeOverlay - wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(runOverlays, ", ")) - err = BuildOverlay(n, "runtime", runOverlays) - if err != nil { - return fmt.Errorf("could not build runtime overlays %v for node %s: %w", runOverlays, n.Id(), err) - } + wg.Wait() + close(errChan) + for err := range errChan { + return err } return nil } -// TODO: Add an Overlay Delete for both sourcedir and image +func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error { + nodeChan := make(chan node.Node, len(nodes)) + errChan := make(chan error, len(nodes)) -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string) error { - for _, n := range nodes { - wwlog.Info("Building overlay for %s: %v", n, overlayNames) - for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) - if err != nil { - return fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + var wg sync.WaitGroup + worker := func() { + for n := range nodeChan { + wwlog.Info("Building overlay for %s: %v", n, overlayNames) + for _, overlayName := range overlayNames { + err := BuildOverlay(n, "", []string{overlayName}) + if err != nil { + errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) + } } } + wg.Done() + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go worker() + } + for _, n := range nodes { + nodeChan <- n + } + close(nodeChan) + + wg.Wait() + close(errChan) + + for err := range errChan { + return err } return nil } @@ -177,7 +214,6 @@ func FindOverlays() (overlayList []string, err error) { overlayList = append(overlayList, file.Name()) } } - return overlayList, nil } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 673d6e68..5e198c4f 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path" + "runtime" "sort" "testing" @@ -522,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes) + err := BuildAllOverlays(nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -614,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays) + err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 973cc6a6..a98c1b47 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -59,9 +59,9 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu if build { if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays) + err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}) + err = overlay.BuildAllOverlays([]node.Node{n}, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", diff --git a/userdocs/contents/overlays.rst b/userdocs/contents/overlays.rst index a71439be..0327dc9c 100644 --- a/userdocs/contents/overlays.rst +++ b/userdocs/contents/overlays.rst @@ -224,24 +224,13 @@ built. Specific overlays can be selected with ``-O`` flag. For debugging purposes the templates can be written to a directory given via the ``-o`` flag. -On clusters with large numbers of nodes a significant speedup can be achieved -by building overlays in parallel. Adding parallel overlay building to `wwctl` -is planned, see issue `#1087 `_. -Until parallel overlay building is implemented, builds can be easily done in -parallel with Gnu `parallel` or `xargs`, for example: +Overlay images for multiple node are built in parallel. By default, each CPU in +the Warewulf server will build overlays independently. The number of workers +can be specified with the ``--workers`` option. -.. code-block:: console - - # Gnu parallel - wwctl node list | awk '{print $1}' | grep -v "NODE " | parallel -j 12 \ - wwctl overlay build -N {} - - # xargs - wwctl node list | awk '{print $1}' | grep -v "NODE " | xargs -n 1 -P 12 \ - wwctl overlay build -N - -By default Warewulf will build/update and cache overlays as needed -(configurable in the ``warewulf.conf``). +Warewulf will attempt to build/update overlays as needed +(configurable in the ``warewulf.conf``); but not all cases are detected, +and manual overlay builds are often necessary. Chmod -----