diff --git a/CHANGELOG.md b/CHANGELOG.md index e40f1419..f088b583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Log cpio errors more prominently. #1615 - Improved syncuser conflict help text. #1614 - Parallelized overlay build. #1018 +- Parallelized and optimized overlay build. #1018 ### Removed diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 2542f3c2..52c461b5 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -97,17 +97,17 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { return fmt.Errorf("could not open node configuration: %s", err) } - nodes, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } - nodes = node.FilterNodeListByName(nodes, []string{nodename}) - if len(nodes) != 1 { + filteredNodes := node.FilterNodeListByName(allNodes, []string{nodename}) + if len(filteredNodes) != 1 { return fmt.Errorf("no single node idendified with %s", nodename) } - overlays := nodes[0].SystemOverlay - overlays = append(overlays, nodes[0].RuntimeOverlay...) - err = overlay.BuildOverlayIndir(nodes[0], overlays, path.Join(runDir, "nodeoverlay")) + overlays := filteredNodes[0].SystemOverlay + overlays = append(overlays, filteredNodes[0].RuntimeOverlay...) + err = overlay.BuildOverlayIndir(filteredNodes[0], allNodes, overlays, path.Join(runDir, "nodeoverlay")) if err != nil { return fmt.Errorf("could not build overlay: %s", err) } diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 21ce42c9..2e987de7 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -18,18 +18,21 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return fmt.Errorf("could not open node configuration: %s", err) } - db, err := nodeDB.FindAllNodes() + allNodes, err := nodeDB.FindAllNodes() if err != nil { return fmt.Errorf("could not get node list: %s", err) } + var filteredNodes []node.Node if len(args) > 0 { args = hostlist.Expand(args) - db = node.FilterNodeListByName(db, args) + filteredNodes = node.FilterNodeListByName(allNodes, args) - if len(db) < len(args) { + if len(filteredNodes) < len(args) { return errors.New("failed to find nodes") } + } else { + filteredNodes = allNodes } // NOTE: this is to keep backward compatible @@ -49,26 +52,25 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if len(args) > 0 { - if len(db) != 1 { + if len(filteredNodes) != 1 { return errors.New("must specify one node to build overlay") } - for _, node := range db { - return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) + for _, node := range filteredNodes { + return overlay.BuildOverlayIndir(node, allNodes, OverlayNames, OverlayDir) } } else { return errors.New("must specify a node to build overlay") } - } oldMask := syscall.Umask(007) defer syscall.Umask(oldMask) if len(OverlayNames) > 0 { - err = overlay.BuildSpecificOverlays(db, OverlayNames, Workers) + err = overlay.BuildSpecificOverlays(filteredNodes, allNodes, OverlayNames, Workers) } else { - err = overlay.BuildAllOverlays(db, Workers) + err = overlay.BuildAllOverlays(filteredNodes, allNodes, Workers) } if err != nil { diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 55d3a208..f53144f8 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -86,7 +86,7 @@ func CobraRunE(cmd *cobra.Command, args []string) (err error) { } } - return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}, Workers) + return overlay.BuildSpecificOverlays(updateNodes, nodes, []string{overlayName}, Workers) } return nil diff --git a/internal/app/wwctl/overlay/show/main.go b/internal/app/wwctl/overlay/show/main.go index 97f7a3e4..533322e7 100644 --- a/internal/app/wwctl/overlay/show/main.go +++ b/internal/app/wwctl/overlay/show/main.go @@ -57,7 +57,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodeConf = node.NewNode(hostName) nodeConf.ClusterName = hostName } - tstruct, err := overlay.InitStruct(overlayName, nodeConf) + var allNodes []node.Node + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + tstruct, err := overlay.InitStruct(overlayName, nodeConf, allNodes) if err != nil { return err } diff --git a/internal/pkg/configure/hostfile.go b/internal/pkg/configure/hostfile.go index 3046eb41..e8bf1759 100644 --- a/internal/pkg/configure/hostfile.go +++ b/internal/pkg/configure/hostfile.go @@ -20,8 +20,18 @@ func Hostfile() (err error) { return fmt.Errorf("'the overlay template '/etc/hosts.ww' does not exists in 'host' overlay") } + var allNodes []node.Node + if nodeDB, err := node.New(); err != nil { + return err + } else { + allNodes, err = nodeDB.FindAllNodes() + if err != nil { + return err + } + } + hostname, _ := os.Hostname() - tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname)) + tstruct, err := overlay.InitStruct(overlay_.Name(), node.NewNode(hostname), allNodes) if err != nil { return err } diff --git a/internal/pkg/overlay/datastructure.go b/internal/pkg/overlay/datastructure.go index 67c61d5f..e818717f 100644 --- a/internal/pkg/overlay/datastructure.go +++ b/internal/pkg/overlay/datastructure.go @@ -46,16 +46,12 @@ type TemplateStruct struct { /* Initialize an TemplateStruct with the given node.NodeInfo */ -func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) { +func InitStruct(overlayName string, nodeData node.Node, allNodes []node.Node) (TemplateStruct, error) { var tstruct TemplateStruct tstruct.Overlay = overlayName hostname, _ := os.Hostname() tstruct.BuildHost = hostname controller := warewulfconf.Get() - nodeDB, err := node.New() - if err != nil { - return tstruct, err - } tstruct.ThisNode = &nodeData if tstruct.ThisNode.Kernel == nil { tstruct.ThisNode.Kernel = new(node.KernelConf) @@ -75,10 +71,6 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) tstruct.Ipaddr6 = controller.Ipaddr6 tstruct.Netmask = controller.Netmask tstruct.Network = controller.Network - allNodes, err := nodeDB.FindAllNodes() - if err != nil { - return tstruct, err - } // init some convenience vars tstruct.Id = nodeData.Id() tstruct.Hostname = nodeData.Id() @@ -92,14 +84,11 @@ func InitStruct(overlayName string, nodeData node.Node) (TemplateStruct, error) var buf bytes.Buffer enc := gob.NewEncoder(&buf) dec := gob.NewDecoder(&buf) - err = enc.Encode(nodeData) - if err != nil { + if err := enc.Encode(nodeData); err != nil { return tstruct, err } - err = dec.Decode(&tstruct) - if err != nil { + if err := dec.Decode(&tstruct); err != nil { return tstruct, err } return tstruct, nil - } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 37724604..e0eba792 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -101,7 +101,7 @@ func (this Overlay) IsDistributionOverlay() bool { return path.Dir(this.Path()) == config.Get().Paths.DistributionOverlaydir() } -func BuildAllOverlays(nodes []node.Node, workerCount int) error { +func BuildAllOverlays(nodes []node.Node, allNodes []node.Node, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)*2) @@ -109,12 +109,12 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { worker := func() { for n := range nodeChan { wwlog.Info("Building system overlays for %s: [%s]", n.Id(), strings.Join(n.SystemOverlay, ", ")) - if err := BuildOverlay(n, "system", n.SystemOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "system", n.SystemOverlay); err != nil { errChan <- fmt.Errorf("could not build system overlays %v for node %s: %w", n.SystemOverlay, n.Id(), err) } wwlog.Info("Building runtime overlays for %s: [%s]", n.Id(), strings.Join(n.RuntimeOverlay, ", ")) - if err := BuildOverlay(n, "runtime", n.RuntimeOverlay); err != nil { + if err := BuildOverlay(n, allNodes, "runtime", n.RuntimeOverlay); err != nil { errChan <- fmt.Errorf("could not build runtime overlays %v for node %s: %w", n.RuntimeOverlay, n.Id(), err) } } @@ -139,7 +139,7 @@ func BuildAllOverlays(nodes []node.Node, workerCount int) error { return nil } -func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount int) error { +func BuildSpecificOverlays(nodes []node.Node, allNodes []node.Node, overlayNames []string, workerCount int) error { nodeChan := make(chan node.Node, len(nodes)) errChan := make(chan error, len(nodes)) @@ -148,7 +148,7 @@ func BuildSpecificOverlays(nodes []node.Node, overlayNames []string, workerCount for n := range nodeChan { wwlog.Info("Building overlay for %s: %v", n, overlayNames) for _, overlayName := range overlayNames { - err := BuildOverlay(n, "", []string{overlayName}) + err := BuildOverlay(n, allNodes, "", []string{overlayName}) if err != nil { errChan <- fmt.Errorf("could not build overlay %s for node %s: %w", overlayName, n.Id(), err) } @@ -190,7 +190,16 @@ func BuildHostOverlay() error { if !(stats.Mode() == os.FileMode(0750|os.ModeDir) || stats.Mode() == os.FileMode(0700|os.ModeDir)) { wwlog.SecWarn("Permissions of host overlay dir %s are %s (750 is considered as secure)", hostdir, stats.Mode()) } - return BuildOverlayIndir(hostData, []string{"host"}, "/") + registry, err := node.New() + if err != nil { + return err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + return err + } + return BuildOverlayIndir(hostData, allNodes, []string{"host"}, "/") } /* @@ -220,7 +229,7 @@ func FindOverlays() (overlayList []string, err error) { /* Build the given overlays for a node and create a Image for them */ -func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) error { +func BuildOverlay(nodeConf node.Node, allNodes []node.Node, context string, overlayNames []string) error { if len(overlayNames) == 0 && context == "" { return nil } @@ -245,7 +254,7 @@ func BuildOverlay(nodeConf node.Node, context string, overlayNames []string) err wwlog.Debug("Created temporary directory for %s: %s", name, buildDir) - err = BuildOverlayIndir(nodeConf, overlayNames, buildDir) + err = BuildOverlayIndir(nodeConf, allNodes, overlayNames, buildDir) if err != nil { return fmt.Errorf("failed to generate files for %s: %w", name, err) } @@ -274,7 +283,7 @@ func init() { } // Build the given overlays for a node in the given directory. -func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir string) error { +func BuildOverlayIndir(nodeData node.Node, allNodes []node.Node, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return nil } @@ -323,7 +332,7 @@ func BuildOverlayIndir(nodeData node.Node, overlayNames []string, outputDir stri } else if filepath.Ext(walkPath) == ".ww" { originalOutputPath := outputPath outputPath := strings.TrimSuffix(outputPath, ".ww") - tstruct, err := InitStruct(overlayName, nodeData) + tstruct, err := InitStruct(overlayName, nodeData, allNodes) if err != nil { return fmt.Errorf("failed to initial data for %s: %w", nodeData.Id(), err) } diff --git a/internal/pkg/overlay/overlay_test.go b/internal/pkg/overlay/overlay_test.go index 5e198c4f..af6a09af 100644 --- a/internal/pkg/overlay/overlay_test.go +++ b/internal/pkg/overlay/overlay_test.go @@ -229,7 +229,7 @@ T3 env.MkdirAll(t, tt.outputDir) - assert.NoError(t, BuildOverlayIndir(tt.node, tt.overlays, env.GetPath(tt.outputDir))) + assert.NoError(t, BuildOverlayIndir(tt.node, []node.Node{tt.node}, tt.overlays, env.GetPath(tt.outputDir))) dirFiles := tt.outputDirs for outputFile, _ := range tt.outputFiles { dirFiles = append(dirFiles, outputFile) @@ -400,7 +400,7 @@ func Test_BuildOverlay(t *testing.T) { for _, tt := range tests { nodeInfo := node.NewNode(tt.nodeName) t.Run(tt.description, func(t *testing.T) { - err := BuildOverlay(nodeInfo, tt.context, tt.overlays) + err := BuildOverlay(nodeInfo, []node.Node{nodeInfo}, tt.context, tt.overlays) assert.NoError(t, err) if tt.image != "" { image := env.GetPath(path.Join("srv/warewulf/overlays", tt.image)) @@ -523,7 +523,7 @@ func Test_BuildAllOverlays(t *testing.T) { } nodes = append(nodes, nodeInfo) } - err := BuildAllOverlays(nodes, runtime.NumCPU()) + err := BuildAllOverlays(nodes, nodes, runtime.NumCPU()) assert.NoError(t, err) if tt.createdOverlays == nil { dirName := path.Join(provisionDir, "overlays") @@ -615,7 +615,7 @@ func Test_BuildSpecificOverlays(t *testing.T) { nodeInfo := node.NewNode(nodeName) nodes = append(nodes, nodeInfo) } - err := BuildSpecificOverlays(nodes, tt.overlays, runtime.NumCPU()) + err := BuildSpecificOverlays(nodes, nodes, tt.overlays, runtime.NumCPU()) if !tt.succeed { assert.Error(t, err) } else { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index a98c1b47..919ee05e 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -58,10 +58,23 @@ func getOverlayFile(n node.Node, context string, stage_overlays []string, autobu } if build { + registry, err := node.New() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } + var allNodes []node.Node + allNodes, err = registry.FindAllNodes() + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + n.Id(), stage_overlays, stage_file, err) + return "", err + } if len(stage_overlays) > 0 { - err = overlay.BuildSpecificOverlays([]node.Node{n}, stage_overlays, 1) + err = overlay.BuildSpecificOverlays([]node.Node{n}, allNodes, stage_overlays, 1) } else { - err = overlay.BuildAllOverlays([]node.Node{n}, 1) + err = overlay.BuildAllOverlays([]node.Node{n}, allNodes, 1) } if err != nil { wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",