Parallel overlay builds

- Closes #1018

Signed-off-by: Jonathon Anderson <janderson@ciq.com>

CHANGELOG.md

@@ -84,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Added note to booting userdoc for removing machine-id. #1609
 - Log cpio errors more prominently. #1615
 - Improved syncuser conflict help text. #1614
+- Parallelized overlay build. #1018
 
 ### Removed
 

internal/app/wwctl/overlay/build/main.go

@@ -66,9 +66,9 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	defer syscall.Umask(oldMask)
 
 	if len(OverlayNames) > 0 {
-		err = overlay.BuildSpecificOverlays(db, OverlayNames)
+		err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers)
 	} else {
-		err = overlay.BuildAllOverlays(db)
+		err = overlay.BuildAllOverlays(db, Workers)
 	}
 
 	if err != nil {

internal/app/wwctl/overlay/build/root.go

@@ -2,6 +2,7 @@ package build
 
 import (
 	"log"
+	"runtime"
 
 	"github.com/spf13/cobra"
 	"github.com/warewulf/warewulf/internal/pkg/node"
@@ -31,6 +32,7 @@ var (
 	}
 	OverlayNames []string
 	OverlayDir   string
+	Workers      int
 )
 
 func init() {
@@ -44,7 +46,7 @@ func init() {
 	}
 	baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to
 	the given directory. An overlay must also be ge given to use this option.`)
-
+	baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays")
 }
 
 // GetRootCommand returns the root cobra.Command for the application.

internal/app/wwctl/overlay/imprt/main.go

@@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) {
 			}
 		}
 
-		return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName})
+		return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers)
 	}
 
 	return nil

internal/app/wwctl/overlay/imprt/root.go

@@ -1,6 +1,8 @@
 package imprt
 
 import (
+	"runtime"
+
 	"github.com/spf13/cobra"
 	"github.com/warewulf/warewulf/internal/pkg/overlay"
 )
@@ -24,11 +26,13 @@ var (
 	}
 	NoOverlayUpdate bool
 	CreateDirs      bool
+	Workers         int
 )
 
 func init() {
 	baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays")
 	baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories")
+	baseCmd.PersistentFlags().IntVar(&Workers, "workers", runtime.NumCPU(), "The number of parallel workers building overlays")
 }
 
 // GetRootCommand returns the root cobra.Command for the application.

internal/pkg/overlay/overlay.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
@@ -100,40 +101,76 @@ func (this Overlay) IsDistributionOverlay() bool {
 	return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir()
 }
 
-/*
-Build all overlays for a node
-*/
-func BuildAllOverlays(nodes []node.Node) error {
+func BuildAllOverlays(nodes []node.Node, workerCount int) error {
+	nodeChan := make(chan node.Node, len(nodes))
+	errChan := make(chan error, len(nodes)*2)
+
+	var wg sync.WaitGroup
+	worker := func() {
+		for n := range nodeChan {
+			wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", "))
+			if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil {
+				errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err)
+			}
+
+			wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", "))
+			if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil {
+				errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err)
+			}
+		}
+		wg.Done()
+	}
+
+	for i := 0; i < workerCount; i++ {
+		wg.Add(1)
+		go worker()
+	}
 	for _, n := range nodes {
+		nodeChan <- n
+	}
+	close(nodeChan)
 
-		sysOverlays := n.SystemOverlay
-		wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(sysOverlays, ", "))
-		err := BuildOverlay(n, "system", sysOverlays)
-		if err != nil {
-			return fmt.Errorf("could not build system overlays %v for node %s: %w", sysOverlays, n.Id(), err)
-		}
-		runOverlays := n.RuntimeOverlay
-		wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(runOverlays, ", "))
-		err = BuildOverlay(n, "runtime", runOverlays)
-		if err != nil {
-			return fmt.Errorf("could not build runtime overlays %v for node %s: %w", runOverlays, n.Id(), err)
-		}
+	wg.Wait()
+	close(errChan)
 
+	for err := range errChan {
+		return err
 	}
 	return nil
 }
 
-// TODO: Add an Overlay Delete for both sourcedir and image
+func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error {
+	nodeChan := make(chan node.Node, len(nodes))
+	errChan := make(chan error, len(nodes))
 
-func BuildSpecificOverlays(nodes []node.Node, overlayNames []string) error {
-	for _, n := range nodes {
-		wwlog.Info("Building overlay for %s: %v", n, overlayNames)
-		for _, overlayName := range overlayNames {
-			err := BuildOverlay(n, "", []string{overlayName})
-			if err != nil {
-				return fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err)
+	var wg sync.WaitGroup
+	worker := func() {
+		for n := range nodeChan {
+			wwlog.Info("Building overlay for %s: %v", n, overlayNames)
+			for _, overlayName := range overlayNames {
+				err := BuildOverlay(n, "", []string{overlayName})
+				if err != nil {
+					errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err)
+				}
 			}
 		}
+		wg.Done()
+	}
+
+	for i := 0; i < workerCount; i++ {
+		wg.Add(1)
+		go worker()
+	}
+	for _, n := range nodes {
+		nodeChan <- n
+	}
+	close(nodeChan)
+
+	wg.Wait()
+	close(errChan)
+
+	for err := range errChan {
+		return err
 	}
 	return nil
 }
@@ -177,7 +214,6 @@ func FindOverlays() (overlayList []string, err error) {
 			overlayList = append(overlayList, file.Name())
 		}
 	}
-
 	return overlayList, nil
 }
 

internal/pkg/overlay/overlay_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"runtime"
 	"sort"
 	"testing"
 
@@ -522,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) {
 				}
 				nodes = append(nodes, nodeInfo)
 			}
-			err := BuildAllOverlays(nodes)
+			err := BuildAllOverlays(nodes, runtime.NumCPU())
 			assert.NoError(t, err)
 			if tt.createdOverlays == nil {
 				dirName := path.Join(provisionDir, "overlays")
@@ -614,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) {
 				nodeInfo := node.NewNode(nodeName)
 				nodes = append(nodes, nodeInfo)
 			}
-			err := BuildSpecificOverlays(nodes, tt.overlays)
+			err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU())
 			if !tt.succeed {
 				assert.Error(t, err)
 			} else {

internal/pkg/warewulfd/util.go

@@ -59,9 +59,9 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu
 
 	if build {
 		if len(stage_overlays) > 0 {
-			err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays)
+			err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1)
 		} else {
-			err = overlay.BuildAllOverlays([]node.Node{n})
+			err = overlay.BuildAllOverlays([]node.Node{n}, 1)
 		}
 		if err != nil {
 			wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",

userdocs/contents/overlays.rst

@@ -224,24 +224,13 @@ built. Specific overlays can be selected with ``-O`` flag. For
 debugging purposes the templates can be written to a directory given
 via the ``-o`` flag.
 
-On clusters with large numbers of nodes a significant speedup can be achieved
-by building overlays in parallel. Adding parallel overlay building to `wwctl`
-is planned, see issue `#1087 <https://github.com/warewulf/warewulf/issues/1087>`_.
-Until parallel overlay building is implemented, builds can be easily done in
-parallel with Gnu `parallel` or `xargs`, for example:
+Overlay images for multiple node are built in parallel. By default, each CPU in
+the Warewulf server will build overlays independently. The number of workers
+can be specified with the ``--workers`` option.
 
-.. code-block:: console
-
-  # Gnu parallel
-  wwctl node list | awk '{print $1}' | grep -v "NODE " | parallel -j 12 \
-    wwctl overlay build -N {}
-
-  # xargs
-  wwctl node list | awk '{print $1}' | grep -v "NODE " | xargs -n 1 -P 12 \
-    wwctl overlay build -N
-
-By default Warewulf will build/update and cache overlays as needed
-(configurable in the ``warewulf.conf``).
+Warewulf will attempt to build/update overlays as needed
+(configurable in the ``warewulf.conf``); but not all cases are detected,
+and manual overlay builds are often necessary.
 
 Chmod
 -----