kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #438 from kcdodd/fix/srvdisco

Browse Source 
Fix server thread safety of discoverable node and overlay build
This commit is contained in:
Gregory M. Kurtzer
2022-06-09 22:18:23 -07:00
committed by GitHub
parent 3cb59b89ea 7eae65e13f
commit 4d44d3a299
8 changed files with 258 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
internal/pkg/warewulfd/nodedb.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
)
type nodeDB struct {
@@ -19,6 +20,12 @@ var (
)
func LoadNodeDB() error {
db.lock.Lock()
defer db.lock.Unlock()
return loadNodeDB()
}
func loadNodeDB() error {
TmpMap := make(map[string]node.NodeInfo)
DB, err := node.New()
@@ -38,8 +45,6 @@ func LoadNodeDB() error {
}
}
db.lock.Lock()
defer db.lock.Unlock()
db.NodeInfo = TmpMap
return nil
@@ -48,6 +53,10 @@ func LoadNodeDB() error {
func GetNode(val string) (node.NodeInfo, error) {
db.lock.RLock()
defer db.lock.RUnlock()
return getNode(val)
}
func getNode(val string) (node.NodeInfo, error) {
if _, ok := db.NodeInfo[val]; ok {
@@ -57,3 +66,69 @@ func GetNode(val string) (node.NodeInfo, error) {
var empty node.NodeInfo
return empty, errors.New("No node found")
}
func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) {
db.lock.Lock()
defer db.lock.Unlock()
return getNodeOrSetDiscoverable(hwaddr)
}
func getNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) {
// NOTE: since discoverable nodes will write an updated DB to file and then
// reload, it is not enough to lock individual reads from the DB
// to ensure the condition on which the node is updated is still satisfied
// after the DB is read back in.
n, err := getNode(hwaddr)
if err == nil {
return n, nil
}
// If we failed to find a node, let's see if we can add one...
var netdev string
wwlog.WarnExc(err, "%s (node not configured)", hwaddr)
config, err := node.New()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to read node configuration file)", hwaddr)
}
_n, netdev, err := config.FindDiscoverableNode()
if err != nil {
// NOTE: this is taken as there is no discoverable node, so return the
// empty one
return n, nil
}
_n.NetDevs[netdev].Hwaddr.Set(hwaddr)
_n.Discoverable.SetB(false)
// NOTE: errors here should return the empty node if the state cannot
// be saved and re-loaded, since subsequent requests will be made on invalid
// assumption that the database is up to date.
err = config.NodeUpdate(_n)
if err != nil {
return n, errors.Wrapf(err, "%s (failed to set node configuration)", hwaddr)
}
err = config.Persist()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr)
}
err = loadNodeDB()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr)
}
// NOTE: previously all overlays were built here, but that will also
// be done automatically when attempting to serve an overlay that
// hasn't been built (without blocking the database).
wwlog.Serv("%s (node automatically configured)", hwaddr)
// return the discovered node
return _n, nil
}

69
internal/pkg/warewulfd/provision.go
View File

@@ -1,17 +1,15 @@
package warewulfd
import (
"bytes"
"net/http"
"path"
"strconv"
"bytes"
"text/template"
"github.com/hpcng/warewulf/internal/pkg/buildconfig"
nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/container"
"github.com/hpcng/warewulf/internal/pkg/kernel"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/warewulfconf"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
@@ -70,47 +68,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
// TODO: when module version is upgraded to go1.18, should be 'any' type
var tmpl_data interface{}
node, err := GetNode(rinfo.hwaddr)
node, err := GetNodeOrSetDiscoverable(rinfo.hwaddr)
if err != nil {
// If we failed to find a node, let's see if we can add one...
var netdev string
wwlog.Warn("%s (node not configured)", rinfo.hwaddr)
nodeDB, err := nodepkg.New()
if err != nil {
wwlog.Error("Could not read node configuration file: %s", err)
w.WriteHeader(http.StatusServiceUnavailable)
return
}
n, netdev, err := nodeDB.FindDiscoverableNode()
if err == nil {
n.NetDevs[netdev].Hwaddr.Set(rinfo.hwaddr)
n.Discoverable.SetB(false)
err := nodeDB.NodeUpdate(n)
if err != nil {
wwlog.Serv("%s (failed to set node configuration)", rinfo.hwaddr)
} else {
err := nodeDB.Persist()
if err != nil {
wwlog.Serv("%s (failed to persist node configuration)", rinfo.hwaddr)
} else {
node = n
_ = overlay.BuildAllOverlays([]nodepkg.NodeInfo{n})
wwlog.Serv("%s (node automatically configured)", rinfo.hwaddr)
err := LoadNodeDB()
if err != nil {
wwlog.Warn("Could not reload configuration: %s", err)
}
}
}
}
wwlog.ErrorExc(err, "")
w.WriteHeader(http.StatusServiceUnavailable)
return
}
if node.AssetKey.Defined() && node.AssetKey.Get() != rinfo.assetkey {
@@ -188,16 +150,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
}
if len(stage_overlays) > 0 {
stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays)
if conf.Warewulf.AutobuildOverlays {
oneoverlaynewer := false
for _, overlayname := range stage_overlays {
oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
}
if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer {
wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays)
_ = overlay.BuildOverlay(node, stage_overlays)
}
stage_file, err = getOverlayFile(
node.Id.Get(),
stage_overlays,
conf.Warewulf.AutobuildOverlays )
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
wwlog.ErrorExc(err, "")
return
}
}
@@ -255,7 +216,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusNotFound)
}
err = sendFile(w, stage_file, node.Id.Get())
err = sendFile(w, req, stage_file, node.Id.Get())
if err != nil {
wwlog.ErrorExc(err, "")
return

85
internal/pkg/warewulfd/util.go
View File

@@ -1,16 +1,21 @@
package warewulfd
import (
"io"
"net/http"
"os"
"strconv"
nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"github.com/pkg/errors"
)
func sendFile(w http.ResponseWriter, filename string, sendto string) error {
func sendFile(
w http.ResponseWriter,
req *http.Request,
filename string,
sendto string) error {
fd, err := os.Open(filename)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
@@ -18,34 +23,60 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) error {
}
defer fd.Close()
FileHeader := make([]byte, 512)
_, err = fd.Read(FileHeader)
stat, err := fd.Stat()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to read header")
return err
}
FileContentType := http.DetectContentType(FileHeader)
FileStat, _ := fd.Stat()
FileSize := strconv.FormatInt(FileStat.Size(), 10)
_, err = fd.Seek(0, 0)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to seek")
}
w.Header().Set("Content-Disposition", "attachment; filename=kernel")
w.Header().Set("Content-Type", FileContentType)
w.Header().Set("Content-Length", FileSize)
_, err = io.Copy(w, fd)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to copy")
}
http.ServeContent(
w,
req,
filename,
stat.ModTime(),
fd )
wwlog.Send("%15s: %s", sendto, filename)
return err
return nil
}
func getOverlayFile(
nodeId string,
stage_overlays []string,
autobuild bool ) (stage_file string, err error) {
stage_file = overlay.OverlayImage(nodeId, stage_overlays)
err = nil
build := !util.IsFile(stage_file)
if !build && autobuild {
build = util.PathIsNewer(stage_file, nodepkg.ConfigFile)
for _, overlayname := range stage_overlays {
build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
}
}
if build {
wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays)
args := []string{"overlay", "build"}
for _, overlayname := range stage_overlays {
args = append(args, "-O", overlayname)
}
args = append(args, nodeId)
out, err := util.RunWWCTL(args...)
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
nodeId, stage_overlays, stage_file, string(out))
}
}
return
}