From c244adc7f69c64c9485b2a24440d11d46c87035d Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Mon, 23 Dec 2024 20:52:06 -0700 Subject: [PATCH] Performance improvements for overlay build Rendering overlay templates requires fetching and populating / merging all other nodes, because overlay templates have access to all cluster node info. This was previously being done for each template render, but is not being done once for each render operation and re-used for each template render. Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 1 + .../app/wwctl/container/exec/child/main.go | 12 ++++---- internal/app/wwctl/overlay/build/main.go | 20 +++++++------ internal/app/wwctl/overlay/imprt/main.go | 2 +- internal/app/wwctl/overlay/show/main.go | 7 ++++- internal/pkg/configure/hostfile.go | 12 +++++++- internal/pkg/overlay/datastructure.go | 17 ++--------- internal/pkg/overlay/overlay.go | 29 ++++++++++++------- internal/pkg/overlay/overlay_test.go | 8 ++--- internal/pkg/warewulfd/util.go | 17 +++++++++-- 10 files changed, 77 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40f1419..f088b583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 - Parallelized overlay build. #1018 +- Parallelized and optimized overlay build. #1018 ### Removed diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 2542f3c2..52c461b5 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -97,17 +97,17 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { return fmt.Errorf("could not open node configuration: %s", err) } - nodes, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } - nodes = node.FilterNodeListByName(nodes, []string{nodename}) - if len(nodes) != 1 { + filteredNodes := node.FilterNodeListByName(allNodes, []string{nodename}) + if len(filteredNodes) != 1 { return fmt.Errorf("no single node idendified with %s", nodename) } - overlays := nodes[0].SystemOverlay - overlays = append(overlays, nodes[0].RuntimeOverlay...) - err = overlay.BuildOverlayIndir(nodes[0], overlays, path.Join(runDir, "nodeoverlay")) + overlays := filteredNodes[0].SystemOverlay + overlays = append(overlays, filteredNodes[0].RuntimeOverlay...) + err = overlay.BuildOverlayIndir(filteredNodes[0], allNodes, overlays, path.Join(runDir, "nodeoverlay")) if err != nil { return fmt.Errorf("could not build overlay: %s", err) } diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 21ce42c9..2e987de7 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -18,18 +18,21 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("could not open node configuration: %s", err) } - db, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } + var filteredNodes []node.Node if len(args) > 0 { args = hostlist.Expand(args) - db = node.FilterNodeListByName(db, args) + filteredNodes = node.FilterNodeListByName(allNodes, args) - if len(db) < len(args) { + if len(filteredNodes) < len(args) { return errors.New("failed to find nodes") } + } else { + filteredNodes = allNodes } // NOTE: this is to keep backward compatible @@ -49,26 +52,25 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if len(args) > 0 { - if len(db) != 1 { + if len(filteredNodes) != 1 { return errors.New("must specify one node to build overlay") } - for _, node := range db { - return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) + for _, node := range filteredNodes { + return overlay.BuildOverlayIndir(node, allNodes, OverlayNames, OverlayDir) } } else { return errors.New("must specify a node to build overlay") } - } oldMask := syscall.Umask(007) defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers) + err = overlay.BuildSpecificOverlays(filteredNodes, allNodes, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db, Workers) + err = overlay.BuildAllOverlays(filteredNodes, allNodes, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 55d3a208..f53144f8 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers) + return overlay.BuildSpecificOverlays(updateNodes, nodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/show/main.go b/internal/app/wwctl/overlay/show/main.go index 97f7a3e4..533322e7 100644 --- a/internal/app/wwctl/overlay/show/main.go +++ b/internal/app/wwctl/overlay/show/main.go @@ -57,7 +57,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodeConf = node.NewNode(hostName) nodeConf.ClusterName = hostName } - tstruct, err := overlay.InitStruct(overlayName, nodeConf) + var allNodes []node.Node + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + tstruct, err := overlay.InitStruct(overlayName, nodeConf, allNodes) if err != nil { return err } diff --git a/internal/pkg/configure/hostfile.go b/internal/pkg/configure/hostfile.go index 3046eb41..e8bf1759 100644 --- a/internal/pkg/configure/hostfile.go +++ b/internal/pkg/configure/hostfile.go @@ -20,8 +20,18 @@ func Hostfile() (err error) { return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay") } + var allNodes []node.Node + if nodeDB, err := node.New(); err != nil { + return err + } else { + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + } + hostname, _ := os.Hostname() - tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname)) + tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname), allNodes) if err != nil { return err } diff --git a/internal/pkg/overlay/datastructure.go b/internal/pkg/overlay/datastructure.go index 67c61d5f..e818717f 100644 --- a/internal/pkg/overlay/datastructure.go +++ b/internal/pkg/overlay/datastructure.go @@ -46,16 +46,12 @@ type TemplateStruct struct { /* Initialize an TemplateStruct with the given node.NodeInfo */ -func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) { +func InitStruct(overlayName string, nodeData node.Node, allNodes []node.Node) (TemplateStruct, error) { var tstruct TemplateStruct tstruct.Overlay = overlayName hostname, _ := os.Hostname() tstruct.BuildHost = hostname controller := warewulfconf.Get() - nodeDB, err := node.New() - if err != nil { - return tstruct, err - } tstruct.ThisNode = &nodeData if tstruct.ThisNode.Kernel == nil { tstruct.ThisNode.Kernel = new(node.KernelConf) @@ -75,10 +71,6 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) tstruct.Ipaddr6 = controller.Ipaddr6 tstruct.Netmask = controller.Netmask tstruct.Network = controller.Network - allNodes, err := nodeDB.FindAllNodes() - if err != nil { - return tstruct, err - } // init some convenience vars tstruct.Id = nodeData.Id() tstruct.Hostname = nodeData.Id() @@ -92,14 +84,11 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) var buf bytes.Buffer enc := gob.NewEncoder(&buf) dec := gob.NewDecoder(&buf) - err = enc.Encode(nodeData) - if err != nil { + if err := enc.Encode(nodeData); err != nil { return tstruct, err } - err = dec.Decode(&tstruct) - if err != nil { + if err := dec.Decode(&tstruct); err != nil { return tstruct, err } return tstruct, nil - } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 37724604..e0eba792 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -101,7 +101,7 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -func BuildAllOverlays(nodes []node.Node, workerCount int) error { +func BuildAllOverlays(nodes []node.Node, allNodes []node.Node, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)*2) @@ -109,12 +109,12 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { worker := func() { for n := range nodeChan { wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) - if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "system", n.SystemOverlay); err != nil { errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) } wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) - if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "runtime", n.RuntimeOverlay); err != nil { errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) } } @@ -139,7 +139,7 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { return nil } -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error { +func BuildSpecificOverlays(nodes []node.Node, allNodes []node.Node, overlayNames []string, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)) @@ -148,7 +148,7 @@ func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount for n := range nodeChan { wwlog.Info("Building overlay for %s: %v", n, overlayNames) for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) + err := BuildOverlay(n, allNodes, "", []string{overlayName}) if err != nil { errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) } @@ -190,7 +190,16 @@ func BuildHostOverlay() error { if !(stats.Mode() == os.FileMode(0750|os.ModeDir) || stats.Mode() == os.FileMode(0700|os.ModeDir)) { wwlog.SecWarn("Permissions of host overlay dir %s are %s (750 is considered as secure)", hostdir, stats.Mode()) } - return BuildOverlayIndir(hostData, []string{"host"}, "/") + registry, err := node.New() + if err != nil { + return err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + return err + } + return BuildOverlayIndir(hostData, allNodes, []string{"host"}, "/") } /* @@ -220,7 +229,7 @@ func FindOverlays() (overlayList []string, err error) { /* Build the given overlays for a node and create a Image for them */ -func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) error { +func BuildOverlay(nodeConf node.Node, allNodes []node.Node, context string, overlayNames []string) error { if len(overlayNames) == 0 && context == "" { return nil } @@ -245,7 +254,7 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err wwlog.Debug("Created temporary directory for %s: %s", name, buildDir) - err = BuildOverlayIndir(nodeConf, overlayNames, buildDir) + err = BuildOverlayIndir(nodeConf, allNodes, overlayNames, buildDir) if err != nil { return fmt.Errorf("failed to generate files for %s: %w", name, err) } @@ -274,7 +283,7 @@ func init() { } // Build the given overlays for a node in the given directory. -func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error { +func BuildOverlayIndir(nodeData node.Node, allNodes []node.Node, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return nil } @@ -323,7 +332,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri } else if filepath.Ext(walkPath) == ".ww" { originalOutputPath := outputPath outputPath := strings.TrimSuffix(outputPath, ".ww") - tstruct, err := InitStruct(overlayName, nodeData) + tstruct, err := InitStruct(overlayName, nodeData, allNodes) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 5e198c4f..af6a09af 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -229,7 +229,7 @@ T3 env.MkdirAll(t, tt.outputDir) - assert.NoError(t, BuildOverlayIndir(tt.node, tt.overlays, env.GetPath(tt.outputDir))) + assert.NoError(t, BuildOverlayIndir(tt.node, []node.Node{tt.node}, tt.overlays, env.GetPath(tt.outputDir))) dirFiles := tt.outputDirs for outputFile, _ := range tt.outputFiles { dirFiles = append(dirFiles, outputFile) @@ -400,7 +400,7 @@ func Test_BuildOverlay(t *testing.T) { for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { - err := BuildOverlay(nodeInfo, tt.context, tt.overlays) + err := BuildOverlay(nodeInfo, []node.Node{nodeInfo}, tt.context, tt.overlays) assert.NoError(t, err) if tt.image != "" { image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image)) @@ -523,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes, runtime.NumCPU()) + err := BuildAllOverlays(nodes, nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -615,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU()) + err := BuildSpecificOverlays(nodes, nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index a98c1b47..919ee05e 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -58,10 +58,23 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu } if build { + registry, err := node.New() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1) + err = overlay.BuildSpecificOverlays([]node.Node{n}, allNodes, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}, 1) + err = overlay.BuildAllOverlays([]node.Node{n}, allNodes, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",