diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 484d249e..56698bff 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -7,7 +7,6 @@ import ( "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" - "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/hpcng/warewulf/pkg/hostlist" @@ -31,63 +30,70 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) os.Exit(1) } - if OverlayDir != "" { - if OverlayName == "" { - return errors.New("no overlay name given") + + if len(args) > 0 { + args = hostlist.Expand(args) + nodes = node.FilterByName(nodes, args) + + if len(nodes) < len(args) { + return errors.New("Failed to find nodes") } + } + + // NOTE: this is to keep backward compatible + // passing -O a,b,c versus -O a -O b -O c, but will also accept -O a,b -O c + overlayNames := []string{} + for _, name := range OverlayNames { + names := strings.Split(name, ",") + overlayNames = append(overlayNames, names...) + } + OverlayNames = overlayNames + + if OverlayDir != "" { + if len(OverlayNames) == 0 { + // TODO: should this behave the same as OverlayDir == "", and build default + // set to overlays? + return errors.New("Must specify overlay(s) to build") + } + if len(args) > 0 { - args = hostlist.Expand(args) + if len(nodes) != 1 { + return errors.New("Must specify one node to build overlay") + } + for _, node := range nodes { - if util.InSlice(node.RuntimeOverlay.GetSlice(), OverlayName) || - util.InSlice(node.SystemOverlay.GetSlice(), OverlayName) { - return overlay.BuildOverlayIndir(node, strings.Split(OverlayName, ","), OverlayDir) - } else { - return errors.New("no node uses the given overlay") - } + return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir) } } else { + // TODO this seems different than what is set in BuildHostOverlay var host node.NodeInfo var idEntry node.Entry hostname, _ := os.Hostname() - wwlog.Printf(wwlog.INFO, "Building overlay for %s: host\n", hostname) + wwlog.Info("Building overlay for host: %s", hostname) idEntry.Set(hostname) host.Id = idEntry - return overlay.BuildOverlayIndir(host, strings.Split(OverlayName, ","), OverlayDir) + return overlay.BuildOverlayIndir(host, OverlayNames, OverlayDir) } } + if BuildHost || (!BuildHost && !BuildNodes && len(args) == 0 && controller.Warewulf.EnableHostOverlay) { err := overlay.BuildHostOverlay() if err != nil { wwlog.Printf(wwlog.WARN, "host overlay could not be built: %s\n", err) } } - if BuildNodes || (!BuildHost && !BuildNodes) { - if len(args) > 0 { - args = hostlist.Expand(args) - if OverlayName != "" { - err = overlay.BuildSpecificOverlays(node.FilterByName(nodes, args), OverlayName) - } else { - err = overlay.BuildAllOverlays(node.FilterByName(nodes, args)) - } + if BuildNodes || (!BuildHost && !BuildNodes) { + if len(OverlayNames) > 0 { + err = overlay.BuildSpecificOverlays(nodes, OverlayNames) } else { - if OverlayName != "" { - for _, n := range nodes { - if util.InSlice(n.RuntimeOverlay.GetSlice(), OverlayName) || - util.InSlice(n.SystemOverlay.GetSlice(), OverlayName) { - err = overlay.BuildSpecificOverlays([]node.NodeInfo{n}, OverlayName) - } - } - } else { - err = overlay.BuildAllOverlays(nodes) - } + err = overlay.BuildAllOverlays(nodes) } if err != nil { - wwlog.Printf(wwlog.WARN, "Some system overlays failed to be generated: %s\n", err) - + wwlog.Printf(wwlog.WARN, "Some overlays failed to be generated: %s\n", err) } } return nil diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index 7437f52b..b9971519 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -31,14 +31,15 @@ var ( } BuildHost bool BuildNodes bool - OverlayName string + OverlayNames []string OverlayDir string ) func init() { baseCmd.PersistentFlags().BoolVarP(&BuildHost, "host", "H", false, "Build overlays only for the host") baseCmd.PersistentFlags().BoolVarP(&BuildNodes, "nodes", "N", false, "Build overlays only for the nodes") - baseCmd.PersistentFlags().StringVarP(&OverlayName, "overlay", "O", "", "Build only specific overlay") + baseCmd.PersistentFlags().StringSliceVarP(&OverlayNames, "overlay", "O", []string{}, "Build only specific overlay(s)") + if err := baseCmd.RegisterFlagCompletionFunc("overlay", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { list, _ := overlay.FindOverlays() return list, cobra.ShellCompDirectiveNoFileComp diff --git a/internal/app/wwctl/overlay/imprt/main.go b/internal/app/wwctl/overlay/imprt/main.go index 549a7c74..0c209087 100644 --- a/internal/app/wwctl/overlay/imprt/main.go +++ b/internal/app/wwctl/overlay/imprt/main.go @@ -70,7 +70,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } - return overlay.BuildSpecificOverlays(updateNodes, overlayName) + return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName}) } return nil diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index df2b317e..2c246ed9 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -68,13 +68,13 @@ func BuildAllOverlays(nodes []node.NodeInfo) error { // TODO: Add an Overlay Delete for both sourcedir and image -func BuildSpecificOverlays(nodes []node.NodeInfo, overlayName string) error { +func BuildSpecificOverlays(nodes []node.NodeInfo, overlayNames []string) error { for _, n := range nodes { - wwlog.Info("Building overlay for %s: %s", n.Id.Get(), overlayName) - err := BuildOverlay(n, []string{overlayName}) + wwlog.Info("Building overlay for %s: %v", n.Id.Get(), overlayNames) + err := BuildOverlay(n, overlayNames) if err != nil { - return errors.Wrap(err, "could not build overlay "+n.Id.Get()+"/"+overlayName+".img") + return errors.Wrapf(err, "could not build overlay for node %s: %v", n.Id.Get(), overlayNames) } } diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index b0b27cc4..05ebfe0d 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -12,7 +12,9 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strings" + "sync/atomic" "syscall" "time" @@ -20,6 +22,45 @@ import ( "github.com/pkg/errors" ) +// reserve some number of cpus for system/warwulfd usage +var processLimitedReserve int = 4 +// maximum number of concurrent spawned processes +var processLimitedMax = MaxInt(1, runtime.NumCPU() - processLimitedReserve) +// Channel used as semaphore to specififed processLimitedMax +var processLimitedChan = make(chan int, processLimitedMax) +// Current number of processes started + queued +var processLimitedNum int32 = 0 +// Counter over total history of started processes +var processLimitedCounter uint32 = 0 + +func ProcessLimitedEnter() (index uint32) { + atomic.AddInt32(&processLimitedNum, 1) + index = atomic.AddUint32(&processLimitedCounter, 1) + // NOTE: blocks when channel is full (i.e. processLimitedMax) + // until a process exists and takes one out of channel + processLimitedChan <- 1 + return +} + +func ProcessLimitedExit() { + <-processLimitedChan + atomic.AddInt32(&processLimitedNum, -1) +} + +func ProcessLimitedStatus() (running int32, queued int32) { + running = int32(len(processLimitedChan)) + queued = processLimitedNum - running + return +} + +func MaxInt( a int, b int ) int { + if a > b { + return a + } + + return b +} + func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -654,3 +695,23 @@ func BuildFsImage( return nil } + +/******************************************************************************* + Runs wwctl command +*/ +func RunWWCTL(args ...string) (out []byte, err error) { + index := ProcessLimitedEnter() + defer ProcessLimitedExit() + running, queued := ProcessLimitedStatus() + + wwlog.Verbose("Starting wwctl process %d (%d running, %d queued): %v", + index, running, queued, args ) + + proc := exec.Command("wwctl", args...) + + out, err = proc.CombinedOutput() + + wwlog.Verbose("Finished wwctl process %d", index) + + return out, err +} diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index 7ece9ea7..96921261 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/wwlog" ) type nodeDB struct { @@ -19,6 +20,12 @@ var ( ) func LoadNodeDB() error { + db.lock.Lock() + defer db.lock.Unlock() + return loadNodeDB() +} + +func loadNodeDB() error { TmpMap := make(map[string]node.NodeInfo) DB, err := node.New() @@ -38,8 +45,6 @@ func LoadNodeDB() error { } } - db.lock.Lock() - defer db.lock.Unlock() db.NodeInfo = TmpMap return nil @@ -48,6 +53,10 @@ func LoadNodeDB() error { func GetNode(val string) (node.NodeInfo, error) { db.lock.RLock() defer db.lock.RUnlock() + return getNode(val) +} + +func getNode(val string) (node.NodeInfo, error) { if _, ok := db.NodeInfo[val]; ok { @@ -57,3 +66,69 @@ func GetNode(val string) (node.NodeInfo, error) { var empty node.NodeInfo return empty, errors.New("No node found") } + +func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { + db.lock.Lock() + defer db.lock.Unlock() + return getNodeOrSetDiscoverable(hwaddr) +} + +func getNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) { + // NOTE: since discoverable nodes will write an updated DB to file and then + // reload, it is not enough to lock individual reads from the DB + // to ensure the condition on which the node is updated is still satisfied + // after the DB is read back in. + + + n, err := getNode(hwaddr) + if err == nil { + return n, nil + } + + // If we failed to find a node, let's see if we can add one... + var netdev string + + wwlog.WarnExc(err, "%s (node not configured)", hwaddr) + + config, err := node.New() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to read node configuration file)", hwaddr) + } + + _n, netdev, err := config.FindDiscoverableNode() + if err != nil { + // NOTE: this is taken as there is no discoverable node, so return the + // empty one + return n, nil + } + + _n.NetDevs[netdev].Hwaddr.Set(hwaddr) + _n.Discoverable.SetB(false) + + // NOTE: errors here should return the empty node if the state cannot + // be saved and re-loaded, since subsequent requests will be made on invalid + // assumption that the database is up to date. + err = config.NodeUpdate(_n) + if err != nil { + return n, errors.Wrapf(err, "%s (failed to set node configuration)", hwaddr) + } + + err = config.Persist() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr) + } + + err = loadNodeDB() + if err != nil { + return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr) + } + + // NOTE: previously all overlays were built here, but that will also + // be done automatically when attempting to serve an overlay that + // hasn't been built (without blocking the database). + + wwlog.Serv("%s (node automatically configured)", hwaddr) + + // return the discovered node + return _n, nil +} diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index 1b12c0c0..a2abe4dd 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -1,17 +1,15 @@ package warewulfd import ( + "bytes" "net/http" "path" "strconv" - "bytes" "text/template" "github.com/hpcng/warewulf/internal/pkg/buildconfig" - nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" @@ -70,47 +68,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { // TODO: when module version is upgraded to go1.18, should be 'any' type var tmpl_data interface{} - node, err := GetNode(rinfo.hwaddr) + node, err := GetNodeOrSetDiscoverable(rinfo.hwaddr) if err != nil { - // If we failed to find a node, let's see if we can add one... - var netdev string - - wwlog.Warn("%s (node not configured)", rinfo.hwaddr) - - nodeDB, err := nodepkg.New() - if err != nil { - wwlog.Error("Could not read node configuration file: %s", err) - w.WriteHeader(http.StatusServiceUnavailable) - return - } - - n, netdev, err := nodeDB.FindDiscoverableNode() - if err == nil { - n.NetDevs[netdev].Hwaddr.Set(rinfo.hwaddr) - n.Discoverable.SetB(false) - err := nodeDB.NodeUpdate(n) - if err != nil { - wwlog.Serv("%s (failed to set node configuration)", rinfo.hwaddr) - - } else { - err := nodeDB.Persist() - if err != nil { - wwlog.Serv("%s (failed to persist node configuration)", rinfo.hwaddr) - - } else { - node = n - _ = overlay.BuildAllOverlays([]nodepkg.NodeInfo{n}) - - wwlog.Serv("%s (node automatically configured)", rinfo.hwaddr) - - err := LoadNodeDB() - if err != nil { - wwlog.Warn("Could not reload configuration: %s", err) - } - - } - } - } + wwlog.ErrorExc(err, "") + w.WriteHeader(http.StatusServiceUnavailable) + return } if node.AssetKey.Defined() && node.AssetKey.Get() != rinfo.assetkey { @@ -188,16 +150,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } if len(stage_overlays) > 0 { - stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays) - if conf.Warewulf.AutobuildOverlays { - oneoverlaynewer := false - for _, overlayname := range stage_overlays { - oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) - } - if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer { - wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays) - _ = overlay.BuildOverlay(node, stage_overlays) - } + stage_file, err = getOverlayFile( + node.Id.Get(), + stage_overlays, + conf.Warewulf.AutobuildOverlays ) + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + wwlog.ErrorExc(err, "") + return } } @@ -255,7 +216,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusNotFound) } - err = sendFile(w, stage_file, node.Id.Get()) + err = sendFile(w, req, stage_file, node.Id.Get()) if err != nil { wwlog.ErrorExc(err, "") return diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 2cc609df..6b2275fe 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -1,16 +1,21 @@ package warewulfd import ( - "io" "net/http" "os" - "strconv" + nodepkg "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "github.com/pkg/errors" ) -func sendFile(w http.ResponseWriter, filename string, sendto string) error { +func sendFile( + w http.ResponseWriter, + req *http.Request, + filename string, + sendto string) error { + fd, err := os.Open(filename) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -18,34 +23,60 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) error { } defer fd.Close() - FileHeader := make([]byte, 512) - _, err = fd.Read(FileHeader) + stat, err := fd.Stat() if err != nil { w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to read header") + return err } - FileContentType := http.DetectContentType(FileHeader) - FileStat, _ := fd.Stat() - FileSize := strconv.FormatInt(FileStat.Size(), 10) - - _, err = fd.Seek(0, 0) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to seek") - } - - w.Header().Set("Content-Disposition", "attachment; filename=kernel") - w.Header().Set("Content-Type", FileContentType) - w.Header().Set("Content-Length", FileSize) - - _, err = io.Copy(w, fd) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return errors.Wrap(err, "failed to copy") - } + http.ServeContent( + w, + req, + filename, + stat.ModTime(), + fd ) wwlog.Send("%15s: %s", sendto, filename) - return err + return nil +} + +func getOverlayFile( + nodeId string, + stage_overlays []string, + autobuild bool ) (stage_file string, err error) { + + stage_file = overlay.OverlayImage(nodeId, stage_overlays) + err = nil + + build := !util.IsFile(stage_file) + + if !build && autobuild { + build = util.PathIsNewer(stage_file, nodepkg.ConfigFile) + + for _, overlayname := range stage_overlays { + build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) + } + } + + if build { + wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays) + + args := []string{"overlay", "build"} + + for _, overlayname := range stage_overlays { + args = append(args, "-O", overlayname) + } + + args = append(args, nodeId) + + out, err := util.RunWWCTL(args...) + + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + nodeId, stage_overlays, stage_file, string(out)) + } + } + + return }