diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index e5367be7..906aa779 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -12,7 +12,9 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strings" + "sync/atomic" "syscall" "time" @@ -20,6 +22,35 @@ import ( "github.com/pkg/errors" ) +// maximum number of concurrent spawned processes +var processLimitedMax = runtime.NumCPU() +// Channel used as semaphore to specififed processLimitedMax +var processLimitedChan = make(chan int, processLimitedMax) +// Current number of processes started + queued +var processLimitedNum int32 = 0 +// Counter over total history of started processes +var processLimitedCounter uint32 = 0 + +func ProcessLimitedEnter() (index int32) { + atomic.AddInt32(&processLimitedNum, 1) + index = atomic.AddUint32(&processLimitedCounter, 1) + // NOTE: blocks when channel is full (i.e. processLimitedMax) + // until a process exists and takes one out of channel + processLimitedChan <- 1 + return +} + +func ProcessLimitedExit() { + <-processLimitedChan + atomic.AddInt32(&processLimitedNum, -1) +} + +func ProcessLimitedStatus() (running int, queued int) { + running = len(processLimitedChan) + queued = processLimitedNum - running + return +} + func FirstError(errs ...error) (err error) { for _, e := range errs { if err == nil { @@ -659,10 +690,18 @@ func BuildFsImage( Runs wwctl command */ func RunWWCTL(args ...string) (out []byte, err error) { + index := ProcessLimitedEnter() + defer ProcessLimitedExit() + running, queued := ProcessLimitedStatus() + + wwlog.Debug("Starting wwctl process %d (%d running, %d queued): %v", + index, running, queued, args ) proc := exec.Command("wwctl", args...) out, err = proc.CombinedOutput() + wwlog.Debug("Finished wwctl process %d", index) + return out, err } diff --git a/internal/pkg/warewulfd/provision.go b/internal/pkg/warewulfd/provision.go index a1416a7b..678574d6 100644 --- a/internal/pkg/warewulfd/provision.go +++ b/internal/pkg/warewulfd/provision.go @@ -1,17 +1,16 @@ package warewulfd import ( + "bytes" "net/http" "path" "strconv" - "bytes" "text/template" "github.com/hpcng/warewulf/internal/pkg/buildconfig" - nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" - "github.com/hpcng/warewulf/internal/pkg/overlay" + nodepkg "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/wwlog" @@ -152,32 +151,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) { } if len(stage_overlays) > 0 { - stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays) - if conf.Warewulf.AutobuildOverlays { - oneoverlaynewer := false - for _, overlayname := range stage_overlays { - oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) - } + stage_file, err = getOverlayFile( + node.Id.Get(), + stage_overlays, + conf.Warewulf.AutobuildOverlays ) - if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer { - wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays) - - args := []string{"overlay", "build"} - - for _, overlayname := range stage_overlays { - args = append(args, "-O", overlayname) - } - - args = append(args, node.Id.Get()) - - out, err := util.RunWWCTL(args...) - - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - wwlog.ErrorExc(err, string(out)) - return - } - } + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + wwlog.ErrorExc(err, "") + return } } diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index 62ad7fa6..6b2275fe 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -4,6 +4,9 @@ import ( "net/http" "os" + nodepkg "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" ) @@ -37,3 +40,43 @@ func sendFile( return nil } + +func getOverlayFile( + nodeId string, + stage_overlays []string, + autobuild bool ) (stage_file string, err error) { + + stage_file = overlay.OverlayImage(nodeId, stage_overlays) + err = nil + + build := !util.IsFile(stage_file) + + if !build && autobuild { + build = util.PathIsNewer(stage_file, nodepkg.ConfigFile) + + for _, overlayname := range stage_overlays { + build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname)) + } + } + + if build { + wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays) + + args := []string{"overlay", "build"} + + for _, overlayname := range stage_overlays { + args = append(args, "-O", overlayname) + } + + args = append(args, nodeId) + + out, err := util.RunWWCTL(args...) + + if err != nil { + wwlog.Error("Failed to build overlay: %s, %s, %s\n%s", + nodeId, stage_overlays, stage_file, string(out)) + } + } + + return +}