Lock server node db when updating discoverable node

internal/pkg/warewulfd/nodedb.go

@@ -7,6 +7,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/overlay"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
 )
 
 type nodeDB struct {
@@ -57,3 +59,64 @@ func GetNode(val string) (node.NodeInfo, error) {
 	var empty node.NodeInfo
 	return empty, errors.New("No node found")
 }
+
+func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) {
+	// NOTE: since discoverable nodes will write an updated DB to file and then
+	// reload, it is not enough to lock individual reads from the DB
+	// to ensure the condition on which the node is updated is still satisfied
+	// after the DB is read back in.
+	db.lock.Lock()
+	defer db.lock.Unlock()
+
+	n, err := GetNode(hwaddr)
+	if err == nil {
+		return n, nil
+	}
+
+	// If we failed to find a node, let's see if we can add one...
+	var netdev string
+
+	wwlog.WarnExc(err, "%s (node not configured)", hwaddr)
+
+	config, err := node.New()
+	if err != nil {
+		return n, errors.Wrapf(err, "%s (failed to read node configuration file)", hwaddr)
+	}
+
+	_n, netdev, err := config.FindDiscoverableNode()
+	if err != nil {
+		// NOTE: this is taken as there is no discoverable node, so return the
+		// empty one
+		return n, nil
+	}
+
+	_n.NetDevs[netdev].Hwaddr.Set(hwaddr)
+	_n.Discoverable.SetB(false)
+
+	// NOTE: errors here should return the empty node if the state cannot
+	// be saved and re-loaded, since subsequent requests will be made on invalid
+	// assumption that the database is up to date.
+	err = config.NodeUpdate(_n)
+	if err != nil {
+		return n, errors.Wrapf(err, "%s (failed to set node configuration)", hwaddr)
+	}
+
+	err = config.Persist()
+	if err != nil {
+		return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr)
+	}
+
+	err = LoadNodeDB()
+	if err != nil {
+		return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr)
+	}
+
+	// NOTE: previously all overlays were built here, but that will also
+	// be done automatically when attempting to serve an overlay that
+	// hasn't been built (without blocking the database).
+
+	wwlog.Serv("%s (node automatically configured)", hwaddr)
+
+	// return the discovered node
+	return _n, nil
+}

internal/pkg/warewulfd/provision.go

@@ -70,47 +70,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
 	// TODO: when module version is upgraded to go1.18, should be 'any' type
 	var tmpl_data interface{}
 
-	node, err := GetNode(rinfo.hwaddr)
+	node, err := GetNodeOrSetDiscoverable(rinfo.hwaddr)
 	if err != nil {
-		// If we failed to find a node, let's see if we can add one...
-		var netdev string
-
-		wwlog.Warn("%s (node not configured)", rinfo.hwaddr)
-
-		nodeDB, err := nodepkg.New()
-		if err != nil {
-			wwlog.Error("Could not read node configuration file: %s", err)
-			w.WriteHeader(http.StatusServiceUnavailable)
-			return
-		}
-
-		n, netdev, err := nodeDB.FindDiscoverableNode()
-		if err == nil {
-			n.NetDevs[netdev].Hwaddr.Set(rinfo.hwaddr)
-			n.Discoverable.SetB(false)
-			err := nodeDB.NodeUpdate(n)
-			if err != nil {
-				wwlog.Serv("%s (failed to set node configuration)", rinfo.hwaddr)
-
-			} else {
-				err := nodeDB.Persist()
-				if err != nil {
-					wwlog.Serv("%s (failed to persist node configuration)", rinfo.hwaddr)
-
-				} else {
-					node = n
-					_ = overlay.BuildAllOverlays([]nodepkg.NodeInfo{n})
-
-					wwlog.Serv("%s (node automatically configured)", rinfo.hwaddr)
-
-					err := LoadNodeDB()
-					if err != nil {
-						wwlog.Warn("Could not reload configuration: %s", err)
-					}
-
-				}
-			}
-		}
+		wwlog.ErrorExc(err, "")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
 	}
 
 	if node.AssetKey.Defined() && node.AssetKey.Get() != rinfo.assetkey {