Move overlay build logic to util, add limit to spawned processes

internal/pkg/util/util.go

@@ -12,7 +12,9 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -20,6 +22,35 @@ import (
 	"github.com/pkg/errors"
 )
 
+// maximum number of concurrent spawned processes
+var processLimitedMax = runtime.NumCPU()
+// Channel used as semaphore to specififed processLimitedMax
+var processLimitedChan = make(chan int, processLimitedMax)
+// Current number of processes started + queued
+var processLimitedNum int32 = 0
+// Counter over total history of started processes
+var processLimitedCounter uint32 = 0
+
+func ProcessLimitedEnter() (index int32) {
+	atomic.AddInt32(&processLimitedNum, 1)
+	index = atomic.AddUint32(&processLimitedCounter, 1)
+	// NOTE: blocks when channel is full (i.e. processLimitedMax)
+	// until a process exists and takes one out of channel
+	processLimitedChan <- 1
+	return
+}
+
+func ProcessLimitedExit() {
+	<-processLimitedChan
+	atomic.AddInt32(&processLimitedNum, -1)
+}
+
+func ProcessLimitedStatus() (running int, queued int) {
+	running = len(processLimitedChan)
+	queued = processLimitedNum - running
+	return
+}
+
 func FirstError(errs ...error) (err error) {
 	for _, e := range errs {
 		if err == nil {
@@ -659,10 +690,18 @@ func BuildFsImage(
 	Runs wwctl command
 */
 func RunWWCTL(args ...string) (out []byte, err error) {
+	index := ProcessLimitedEnter()
+	defer ProcessLimitedExit()
+	running, queued := ProcessLimitedStatus()
+
+	wwlog.Debug("Starting wwctl process %d (%d running, %d queued): %v",
+		index, running, queued, args )
 
 	proc := exec.Command("wwctl", args...)
 
 	out, err = proc.CombinedOutput()
 
+	wwlog.Debug("Finished wwctl process %d", index)
+
 	return out, err
 }

internal/pkg/warewulfd/provision.go

@@ -1,17 +1,16 @@
 package warewulfd
 
 import (
+	"bytes"
 	"net/http"
 	"path"
 	"strconv"
-	"bytes"
 	"text/template"
 
 	"github.com/hpcng/warewulf/internal/pkg/buildconfig"
-	nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
 	"github.com/hpcng/warewulf/internal/pkg/container"
 	"github.com/hpcng/warewulf/internal/pkg/kernel"
-	"github.com/hpcng/warewulf/internal/pkg/overlay"
+	nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
 	"github.com/hpcng/warewulf/internal/pkg/util"
 	"github.com/hpcng/warewulf/internal/pkg/warewulfconf"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
@@ -152,32 +151,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if len(stage_overlays) > 0 {
-		stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays)
-		if conf.Warewulf.AutobuildOverlays {
-			oneoverlaynewer := false
-			for _, overlayname := range stage_overlays {
-				oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
-			}
+		stage_file, err = getOverlayFile(
+			node.Id.Get(),
+			stage_overlays,
+			conf.Warewulf.AutobuildOverlays )
 
-			if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer {
-				wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays)
-
-				args := []string{"overlay", "build"}
-
-				for _, overlayname := range stage_overlays {
-					args = append(args, "-O", overlayname)
-				}
-
-				args = append(args, node.Id.Get())
-
-				out, err := util.RunWWCTL(args...)
-
-				if err != nil {
-					w.WriteHeader(http.StatusInternalServerError)
-					wwlog.ErrorExc(err, string(out))
-					return
-				}
-			}
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			wwlog.ErrorExc(err, "")
+			return
 		}
 	}
 

internal/pkg/warewulfd/util.go

@@ -4,6 +4,9 @@ import (
 	"net/http"
 	"os"
 
+	nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/overlay"
+	"github.com/hpcng/warewulf/internal/pkg/util"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
 )
 
@@ -37,3 +40,43 @@ func sendFile(
 
 	return nil
 }
+
+func getOverlayFile(
+	nodeId string,
+	stage_overlays []string,
+	autobuild bool ) (stage_file string, err error) {
+
+	stage_file = overlay.OverlayImage(nodeId, stage_overlays)
+	err = nil
+
+	build := !util.IsFile(stage_file)
+
+	if !build && autobuild {
+		build = util.PathIsNewer(stage_file, nodepkg.ConfigFile)
+
+		for _, overlayname := range stage_overlays {
+			build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
+		}
+	}
+
+	if build {
+		wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays)
+
+		args := []string{"overlay", "build"}
+
+		for _, overlayname := range stage_overlays {
+			args = append(args, "-O", overlayname)
+		}
+
+		args = append(args, nodeId)
+
+		out, err := util.RunWWCTL(args...)
+
+		if err != nil {
+			wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
+				nodeId, stage_overlays, stage_file, string(out))
+		}
+	}
+
+	return
+}