From 95ca053960f5675597f7bf5d5667a42a7d6a8847 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Thu, 2 Jun 2022 20:03:10 -0500 Subject: [PATCH 1/9] Lock server node db when updating discoverable node --- internal/pkg/warewulfd/nodedb.go | 63 +++++++++++++++++++++++++++++ internal/pkg/warewulfd/provision.go | 44 ++------------------ 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index 7ece9ea7..e840a516 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -7,6 +7,8 @@ import ( "github.com/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/wwlog" ) type nodeDB struct { @@ -57,3 +59,64 @@ func GetNode(val string) (node.NodeInfo, error) { var empty node.NodeInfo return empty, errors.New("No node found") } + +func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { + // NOTE: since discoverable nodes will write an updated DB to file and then + // reload, it is not enough to lock individual reads from the DB + // to ensure the condition on which the node is updated is still satisfied + // after the DB is read back in. + db.lock.Lock() + defer db.lock.Unlock() + + n, err := GetNode(hwaddr) + if err == nil { + return n, nil + } + + // If we failed to find a node, let's see if we can add one... + var netdev string + + wwlog.WarnExc(err, "%s (node not configured)", hwaddr) + + config, err := node.New() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to read node configuration file)", hwaddr) + } + + _n, netdev, err := config.FindDiscoverableNode() + if err != nil { + // NOTE: this is taken as there is no discoverable node, so return the + // empty one + return n, nil + } + + _n.NetDevs[netdev].Hwaddr.Set(hwaddr) + _n.Discoverable.SetB(false) + + // NOTE: errors here should return the empty node if the state cannot + // be saved and re-loaded, since subsequent requests will be made on invalid + // assumption that the database is up to date. + err = config.NodeUpdate(_n) + if err != nil { + return n, errors.Wrapf(err, "%s (failed to set node configuration)", hwaddr) + } + + err = config.Persist() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr) + } + + err = LoadNodeDB() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr) + } + + // NOTE: previously all overlays were built here, but that will also + // be done automatically when attempting to serve an overlay that + // hasn't been built (without blocking the database). + + wwlog.Serv("%s (node automatically configured)", hwaddr) + + // return the discovered node + return _n, nil +} diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index c1658a39..1fd8270b 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -70,47 +70,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { // TODO: when module version is upgraded to go1.18, should be 'any' type var tmpl_data interface{} - node, err := GetNode(rinfo.hwaddr) + node, err := GetNodeOrSetDiscoverable(rinfo.hwaddr) if err != nil { - // If we failed to find a node, let's see if we can add one... - var netdev string - - wwlog.Warn("%s (node not configured)", rinfo.hwaddr) - - nodeDB, err := nodepkg.New() - if err != nil { - wwlog.Error("Could not read node configuration file: %s", err) - w.WriteHeader(http.StatusServiceUnavailable) - return - } - - n, netdev, err := nodeDB.FindDiscoverableNode() - if err == nil { - n.NetDevs[netdev].Hwaddr.Set(rinfo.hwaddr) - n.Discoverable.SetB(false) - err := nodeDB.NodeUpdate(n) - if err != nil { - wwlog.Serv("%s (failed to set node configuration)", rinfo.hwaddr) - - } else { - err := nodeDB.Persist() - if err != nil { - wwlog.Serv("%s (failed to persist node configuration)", rinfo.hwaddr) - - } else { - node = n - _ = overlay.BuildAllOverlays([]nodepkg.NodeInfo{n}) - - wwlog.Serv("%s (node automatically configured)", rinfo.hwaddr) - - err := LoadNodeDB() - if err != nil { - wwlog.Warn("Could not reload configuration: %s", err) - } - - } - } - } + wwlog.ErrorExc(err, "") + w.WriteHeader(http.StatusServiceUnavailable) + return } if node.AssetKey.Defined() && node.AssetKey.Get() != rinfo.assetkey { From 04fb854aa745cd24777d9cdb230360d640582ed5 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Thu, 2 Jun 2022 20:20:03 -0500 Subject: [PATCH 2/9] fixup import --- internal/pkg/warewulfd/nodedb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index e840a516..cb851a05 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/wwlog" ) From 7caabc439d6e4ec75310fb1eaab52213341a4c04 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Thu, 2 Jun 2022 22:22:19 -0500 Subject: [PATCH 3/9] separate safe/unsafe db operations --- internal/pkg/warewulfd/nodedb.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index cb851a05..96921261 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -20,6 +20,12 @@ var ( ) func LoadNodeDB() error { + db.lock.Lock() + defer db.lock.Unlock() + return loadNodeDB() +} + +func loadNodeDB() error { TmpMap := make(map[string]node.NodeInfo) DB, err := node.New() @@ -39,8 +45,6 @@ func LoadNodeDB() error { } } - db.lock.Lock() - defer db.lock.Unlock() db.NodeInfo = TmpMap return nil @@ -49,6 +53,10 @@ func LoadNodeDB() error { func GetNode(val string) (node.NodeInfo, error) { db.lock.RLock() defer db.lock.RUnlock() + return getNode(val) +} + +func getNode(val string) (node.NodeInfo, error) { if _, ok := db.NodeInfo[val]; ok { @@ -60,14 +68,19 @@ func GetNode(val string) (node.NodeInfo, error) { } func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { + db.lock.Lock() + defer db.lock.Unlock() + return getNodeOrSetDiscoverable(hwaddr) +} + +func getNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { // NOTE: since discoverable nodes will write an updated DB to file and then // reload, it is not enough to lock individual reads from the DB // to ensure the condition on which the node is updated is still satisfied // after the DB is read back in. - db.lock.Lock() - defer db.lock.Unlock() - n, err := GetNode(hwaddr) + + n, err := getNode(hwaddr) if err == nil { return n, nil } @@ -105,7 +118,7 @@ func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr) } - err = LoadNodeDB() + err = loadNodeDB() if err != nil { return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr) } From fc8d3863d82778c1724a3dec5ee0cb4e8d04b3ac Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sat, 4 Jun 2022 11:14:48 -0500 Subject: [PATCH 4/9] use wwctl to re-build provision overlays, update to pass overlaynames --- internal/app/wwctl/overlay/build/main.go | 62 ++++++++++++------------ internal/app/wwctl/overlay/build/root.go | 5 +- internal/app/wwctl/overlay/imprt/main.go | 2 +- internal/pkg/overlay/overlay.go | 8 +-- internal/pkg/util/util.go | 12 +++++ internal/pkg/warewulfd/provision.go | 20 +++++++- internal/pkg/warewulfd/util.go | 42 ++++++---------- 7 files changed, 83 insertions(+), 68 deletions(-) diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 484d249e..4477e6ef 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -31,63 +31,61 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) os.Exit(1) } - if OverlayDir != "" { - if OverlayName == "" { - return errors.New("no overlay name given") + + if len(args) > 0 { + args = hostlist.Expand(args) + nodes = node.FilterByName(nodes, args) + + if len(nodes) < len(args) { + return errors.New("Failed to find nodes") } + } + + if OverlayDir != "" { + if len(OverlayNames) == 0 { + // TODO: should this behave the same as OverlayDir == "", and build default + // set to overlays? + return errors.New("Must specify overlay(s) to build") + } + if len(args) > 0 { - args = hostlist.Expand(args) + if len(nodes) != 1 { + return errors.New("Must specify one node to build overlay") + } + for _, node := range nodes { - if util.InSlice(node.RuntimeOverlay.GetSlice(), OverlayName) || - util.InSlice(node.SystemOverlay.GetSlice(), OverlayName) { - return overlay.BuildOverlayIndir(node, strings.Split(OverlayName, ","), OverlayDir) - } else { - return errors.New("no node uses the given overlay") - } + return overlay.BuildOverlayIndir(node, OverlayNames), OverlayDir) } } else { + // TODO this seems different than what is set in BuildHostOverlay var host node.NodeInfo var idEntry node.Entry hostname, _ := os.Hostname() - wwlog.Printf(wwlog.INFO, "Building overlay for %s: host\n", hostname) + wwlog.Info("Building overlay for host: %s", hostname) idEntry.Set(hostname) host.Id = idEntry - return overlay.BuildOverlayIndir(host, strings.Split(OverlayName, ","), OverlayDir) + return overlay.BuildOverlayIndir(host, OverlayNames, OverlayDir) } } + if BuildHost || (!BuildHost && !BuildNodes && len(args) == 0 && controller.Warewulf.EnableHostOverlay) { err := overlay.BuildHostOverlay() if err != nil { wwlog.Printf(wwlog.WARN, "host overlay could not be built: %s\n", err) } } - if BuildNodes || (!BuildHost && !BuildNodes) { - if len(args) > 0 { - args = hostlist.Expand(args) - if OverlayName != "" { - err = overlay.BuildSpecificOverlays(node.FilterByName(nodes, args), OverlayName) - } else { - err = overlay.BuildAllOverlays(node.FilterByName(nodes, args)) - } + if BuildNodes || (!BuildHost && !BuildNodes) { + if len(OverlayNames) > 0 { + err = overlay.BuildSpecificOverlays(nodes, OverlayNames) } else { - if OverlayName != "" { - for _, n := range nodes { - if util.InSlice(n.RuntimeOverlay.GetSlice(), OverlayName) || - util.InSlice(n.SystemOverlay.GetSlice(), OverlayName) { - err = overlay.BuildSpecificOverlays([]node.NodeInfo{n}, OverlayName) - } - } - } else { - err = overlay.BuildAllOverlays(nodes) - } + err = overlay.BuildAllOverlays(nodes) } if err != nil { - wwlog.Printf(wwlog.WARN, "Some system overlays failed to be generated: %s\n", err) - + wwlog.Printf(wwlog.WARN, "Some overlays failed to be generated: %s\n", err) } } return nil diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index 7437f52b..958a9c92 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -31,14 +31,15 @@ var ( } BuildHost bool BuildNodes bool - OverlayName string + OverlayNames []string OverlayDir string ) func init() { baseCmd.PersistentFlags().BoolVarP(&BuildHost, "host", "H", false, "Build overlays only for the host") baseCmd.PersistentFlags().BoolVarP(&BuildNodes, "nodes", "N", false, "Build overlays only for the nodes") - baseCmd.PersistentFlags().StringVarP(&OverlayName, "overlay", "O", "", "Build only specific overlay") + baseCmd.PersistentFlags().StringSliceVar(&OverlayNames, "overlay", "O", []string{}, "Build only specific overlay(s)") + if err := baseCmd.RegisterFlagCompletionFunc("overlay", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, _ := overlay.FindOverlays() return list, cobra.ShellCompDirectiveNoFileComp diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 549a7c74..0c209087 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -70,7 +70,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } - return overlay.BuildSpecificOverlays(updateNodes, overlayName) + return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}) } return nil diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 634b5082..b877a29f 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -68,13 +68,13 @@ func BuildAllOverlays(nodes []node.NodeInfo) error { // TODO: Add an Overlay Delete for both sourcedir and image -func BuildSpecificOverlays(nodes []node.NodeInfo, overlayName string) error { +func BuildSpecificOverlays(nodes []node.NodeInfo, overlayNames []string) error { for _, n := range nodes { - wwlog.Info("Building overlay for %s: %s", n.Id.Get(), overlayName) - err := BuildOverlay(n, []string{overlayName}) + wwlog.Info("Building overlay for %s: %v", n.Id.Get(), overlayNames) + err := BuildOverlay(n, overlayNames) if err != nil { - return errors.Wrap(err, "could not build overlay "+n.Id.Get()+"/"+overlayName+".img") + return errors.Wrapf(err, "could not build overlay for node %s: %v", n.Id.Get(), overlayNames) } } diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index b0b27cc4..9b7aca45 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -654,3 +654,15 @@ func BuildFsImage( return nil } + +/******************************************************************************* + Runs wwctl command +*/ +func RunWWCTL(args ...string) (out string, err error) { + + proc := exec.Command("wwctl", args...) + + out, err := proc.CombinedOutput() + + return out, err +} diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 1fd8270b..7cd2bfca 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -158,9 +158,25 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { for _, overlayname := range stage_overlays { oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) } + if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer { wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays) - _ = overlay.BuildOverlay(node, stage_overlays) + + args := []string{} + + for _, overlayname := range stage_overlays { + args = append(args, "-O", overlayname) + } + + args = append(args, node.Id.Get()) + + out, err := util.RunWWCTL("overlay", "build", args...) + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + wwlog.ErrorExc(err, out) + return + } } } } @@ -219,7 +235,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusNotFound) } - err = sendFile(w, stage_file, node.Id.Get()) + err = sendFile(w, req, stage_file, node.Id.Get()) if err != nil { wwlog.ErrorExc(err, "") return diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 2cc609df..62ad7fa6 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -1,16 +1,18 @@ package warewulfd import ( - "io" "net/http" "os" - "strconv" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "github.com/pkg/errors" ) -func sendFile(w http.ResponseWriter, filename string, sendto string) error { +func sendFile( + w http.ResponseWriter, + req *http.Request, + filename string, + sendto string) error { + fd, err := os.Open(filename) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -18,34 +20,20 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) error { } defer fd.Close() - FileHeader := make([]byte, 512) - _, err = fd.Read(FileHeader) + stat, err := fd.Stat() if err != nil { w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to read header") + return err } - FileContentType := http.DetectContentType(FileHeader) - FileStat, _ := fd.Stat() - FileSize := strconv.FormatInt(FileStat.Size(), 10) - - _, err = fd.Seek(0, 0) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to seek") - } - - w.Header().Set("Content-Disposition", "attachment; filename=kernel") - w.Header().Set("Content-Type", FileContentType) - w.Header().Set("Content-Length", FileSize) - - _, err = io.Copy(w, fd) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to copy") - } + http.ServeContent( + w, + req, + filename, + stat.ModTime(), + fd ) wwlog.Send("%15s: %s", sendto, filename) - return err + return nil } From f14f33957649e718a90d69512d70ba40c0f3ac0f Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sat, 4 Jun 2022 11:42:16 -0500 Subject: [PATCH 5/9] fixups --- internal/app/wwctl/overlay/build/main.go | 4 +--- internal/app/wwctl/overlay/build/root.go | 2 +- internal/pkg/util/util.go | 4 ++-- internal/pkg/warewulfd/provision.go | 6 +++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 4477e6ef..324cfc3e 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -3,11 +3,9 @@ package build import ( "errors" "os" - "strings" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" - "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/hpcng/warewulf/pkg/hostlist" @@ -54,7 +52,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } for _, node := range nodes { - return overlay.BuildOverlayIndir(node, OverlayNames), OverlayDir) + return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) } } else { // TODO this seems different than what is set in BuildHostOverlay diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index 958a9c92..b9971519 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -38,7 +38,7 @@ var ( func init() { baseCmd.PersistentFlags().BoolVarP(&BuildHost, "host", "H", false, "Build overlays only for the host") baseCmd.PersistentFlags().BoolVarP(&BuildNodes, "nodes", "N", false, "Build overlays only for the nodes") - baseCmd.PersistentFlags().StringSliceVar(&OverlayNames, "overlay", "O", []string{}, "Build only specific overlay(s)") + baseCmd.PersistentFlags().StringSliceVarP(&OverlayNames, "overlay", "O", []string{}, "Build only specific overlay(s)") if err := baseCmd.RegisterFlagCompletionFunc("overlay", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, _ := overlay.FindOverlays() diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 9b7aca45..e5367be7 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -658,11 +658,11 @@ func BuildFsImage( /******************************************************************************* Runs wwctl command */ -func RunWWCTL(args ...string) (out string, err error) { +func RunWWCTL(args ...string) (out []byte, err error) { proc := exec.Command("wwctl", args...) - out, err := proc.CombinedOutput() + out, err = proc.CombinedOutput() return out, err } diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 7cd2bfca..a1416a7b 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -162,7 +162,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer { wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays) - args := []string{} + args := []string{"overlay", "build"} for _, overlayname := range stage_overlays { args = append(args, "-O", overlayname) @@ -170,11 +170,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { args = append(args, node.Id.Get()) - out, err := util.RunWWCTL("overlay", "build", args...) + out, err := util.RunWWCTL(args...) if err != nil { w.WriteHeader(http.StatusInternalServerError) - wwlog.ErrorExc(err, out) + wwlog.ErrorExc(err, string(out)) return } } From 97c24e22f7b2d1712c791af7706aed87df1d7219 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sat, 4 Jun 2022 12:13:13 -0500 Subject: [PATCH 6/9] fixup for backward compatible case --- internal/app/wwctl/overlay/build/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 324cfc3e..56698bff 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -3,6 +3,7 @@ package build import ( "errors" "os" + "strings" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" @@ -39,6 +40,15 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } + // NOTE: this is to keep backward compatible + // passing -O a,b,c versus -O a -O b -O c, but will also accept -O a,b -O c + overlayNames := []string{} + for _, name := range OverlayNames { + names := strings.Split(name, ",") + overlayNames = append(overlayNames, names...) + } + OverlayNames = overlayNames + if OverlayDir != "" { if len(OverlayNames) == 0 { // TODO: should this behave the same as OverlayDir == "", and build default From 30ba916703c89ff016c909017d6f0d8b909fb0b6 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sun, 5 Jun 2022 17:28:14 -0500 Subject: [PATCH 7/9] Move overlay build logic to util, add limit to spawned processes --- internal/pkg/util/util.go | 39 ++++++++++++++++++++++++++ internal/pkg/warewulfd/provision.go | 38 +++++++------------------ internal/pkg/warewulfd/util.go | 43 +++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index e5367be7..906aa779 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -12,7 +12,9 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strings" + "sync/atomic" "syscall" "time" @@ -20,6 +22,35 @@ import ( "github.com/pkg/errors" ) +// maximum number of concurrent spawned processes +var processLimitedMax = runtime.NumCPU() +// Channel used as semaphore to specififed processLimitedMax +var processLimitedChan = make(chan int, processLimitedMax) +// Current number of processes started + queued +var processLimitedNum int32 = 0 +// Counter over total history of started processes +var processLimitedCounter uint32 = 0 + +func ProcessLimitedEnter() (index int32) { + atomic.AddInt32(&processLimitedNum, 1) + index = atomic.AddUint32(&processLimitedCounter, 1) + // NOTE: blocks when channel is full (i.e. processLimitedMax) + // until a process exists and takes one out of channel + processLimitedChan <- 1 + return +} + +func ProcessLimitedExit() { + <-processLimitedChan + atomic.AddInt32(&processLimitedNum, -1) +} + +func ProcessLimitedStatus() (running int, queued int) { + running = len(processLimitedChan) + queued = processLimitedNum - running + return +} + func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -659,10 +690,18 @@ func BuildFsImage( Runs wwctl command */ func RunWWCTL(args ...string) (out []byte, err error) { + index := ProcessLimitedEnter() + defer ProcessLimitedExit() + running, queued := ProcessLimitedStatus() + + wwlog.Debug("Starting wwctl process %d (%d running, %d queued): %v", + index, running, queued, args ) proc := exec.Command("wwctl", args...) out, err = proc.CombinedOutput() + wwlog.Debug("Finished wwctl process %d", index) + return out, err } diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index a1416a7b..678574d6 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -1,17 +1,16 @@ package warewulfd import ( + "bytes" "net/http" "path" "strconv" - "bytes" "text/template" "github.com/hpcng/warewulf/internal/pkg/buildconfig" - nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" - "github.com/hpcng/warewulf/internal/pkg/overlay" + nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" @@ -152,32 +151,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } if len(stage_overlays) > 0 { - stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays) - if conf.Warewulf.AutobuildOverlays { - oneoverlaynewer := false - for _, overlayname := range stage_overlays { - oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) - } + stage_file, err = getOverlayFile( + node.Id.Get(), + stage_overlays, + conf.Warewulf.AutobuildOverlays ) - if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer { - wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays) - - args := []string{"overlay", "build"} - - for _, overlayname := range stage_overlays { - args = append(args, "-O", overlayname) - } - - args = append(args, node.Id.Get()) - - out, err := util.RunWWCTL(args...) - - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - wwlog.ErrorExc(err, string(out)) - return - } - } + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + wwlog.ErrorExc(err, "") + return } } diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 62ad7fa6..6b2275fe 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -4,6 +4,9 @@ import ( "net/http" "os" + nodepkg "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" ) @@ -37,3 +40,43 @@ func sendFile( return nil } + +func getOverlayFile( + nodeId string, + stage_overlays []string, + autobuild bool ) (stage_file string, err error) { + + stage_file = overlay.OverlayImage(nodeId, stage_overlays) + err = nil + + build := !util.IsFile(stage_file) + + if !build && autobuild { + build = util.PathIsNewer(stage_file, nodepkg.ConfigFile) + + for _, overlayname := range stage_overlays { + build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) + } + } + + if build { + wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays) + + args := []string{"overlay", "build"} + + for _, overlayname := range stage_overlays { + args = append(args, "-O", overlayname) + } + + args = append(args, nodeId) + + out, err := util.RunWWCTL(args...) + + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + nodeId, stage_overlays, stage_file, string(out)) + } + } + + return +} From 3242d2998e7213fa52b72ab0a4a411fd0fdba3af Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sun, 5 Jun 2022 17:37:23 -0500 Subject: [PATCH 8/9] fixups --- internal/pkg/util/util.go | 6 +++--- internal/pkg/warewulfd/provision.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 906aa779..811e39ed 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -31,7 +31,7 @@ var processLimitedNum int32 = 0 // Counter over total history of started processes var processLimitedCounter uint32 = 0 -func ProcessLimitedEnter() (index int32) { +func ProcessLimitedEnter() (index uint32) { atomic.AddInt32(&processLimitedNum, 1) index = atomic.AddUint32(&processLimitedCounter, 1) // NOTE: blocks when channel is full (i.e. processLimitedMax) @@ -45,8 +45,8 @@ func ProcessLimitedExit() { atomic.AddInt32(&processLimitedNum, -1) } -func ProcessLimitedStatus() (running int, queued int) { - running = len(processLimitedChan) +func ProcessLimitedStatus() (running int32, queued int32) { + running = int32(len(processLimitedChan)) queued = processLimitedNum - running return } diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 678574d6..156ab3f1 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -10,7 +10,6 @@ import ( "github.com/hpcng/warewulf/internal/pkg/buildconfig" "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" - nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" From 7eae65e13f83bbdd08710b4cb551f98855df5c2e Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Mon, 6 Jun 2022 17:43:33 -0500 Subject: [PATCH 9/9] Add process max reserve --- internal/pkg/util/util.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 811e39ed..05ebfe0d 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" ) +// reserve some number of cpus for system/warwulfd usage +var processLimitedReserve int = 4 // maximum number of concurrent spawned processes -var processLimitedMax = runtime.NumCPU() +var processLimitedMax = MaxInt(1, runtime.NumCPU() - processLimitedReserve) // Channel used as semaphore to specififed processLimitedMax var processLimitedChan = make(chan int, processLimitedMax) // Current number of processes started + queued @@ -51,6 +53,14 @@ func ProcessLimitedStatus() (running int32, queued int32) { return } +func MaxInt( a int, b int ) int { + if a > b { + return a + } + + return b +} + func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -694,14 +704,14 @@ func RunWWCTL(args ...string) (out []byte, err error) { defer ProcessLimitedExit() running, queued := ProcessLimitedStatus() - wwlog.Debug("Starting wwctl process %d (%d running, %d queued): %v", + wwlog.Verbose("Starting wwctl process %d (%d running, %d queued): %v", index, running, queued, args ) proc := exec.Command("wwctl", args...) out, err = proc.CombinedOutput() - wwlog.Debug("Finished wwctl process %d", index) + wwlog.Verbose("Finished wwctl process %d", index) return out, err }