kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #438 from kcdodd/fix/srvdisco

Browse Source 
Fix server thread safety of discoverable node and overlay build
This commit is contained in:
Gregory M. Kurtzer
2022-06-09 22:18:23 -07:00
committed by GitHub
parent 3cb59b89ea 7eae65e13f
commit 4d44d3a299
8 changed files with 258 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
internal/app/wwctl/overlay/build/main.go
View File

@@ -7,7 +7,6 @@ import (
"github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/warewulfconf"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"github.com/hpcng/warewulf/pkg/hostlist"
@@ -31,63 +30,70 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err)
os.Exit(1)
}
if OverlayDir != "" {
if OverlayName == "" {
return errors.New("no overlay name given")
if len(args) > 0 {
args = hostlist.Expand(args)
nodes = node.FilterByName(nodes, args)
if len(nodes) < len(args) {
return errors.New("Failed to find nodes")
}
}
// NOTE: this is to keep backward compatible
// passing -O a,b,c versus -O a -O b -O c, but will also accept -O a,b -O c
overlayNames := []string{}
for _, name := range OverlayNames {
names := strings.Split(name, ",")
overlayNames = append(overlayNames, names...)
}
OverlayNames = overlayNames
if OverlayDir != "" {
if len(OverlayNames) == 0 {
// TODO: should this behave the same as OverlayDir == "", and build default
// set to overlays?
return errors.New("Must specify overlay(s) to build")
}
if len(args) > 0 {
args = hostlist.Expand(args)
if len(nodes) != 1 {
return errors.New("Must specify one node to build overlay")
}
for _, node := range nodes {
if util.InSlice(node.RuntimeOverlay.GetSlice(), OverlayName) ||
util.InSlice(node.SystemOverlay.GetSlice(), OverlayName) {
return overlay.BuildOverlayIndir(node, strings.Split(OverlayName, ","), OverlayDir)
} else {
return errors.New("no node uses the given overlay")
}
return overlay.BuildOverlayIndir(node, OverlayNames, OverlayDir)
}
} else {
// TODO this seems different than what is set in BuildHostOverlay
var host node.NodeInfo
var idEntry node.Entry
hostname, _ := os.Hostname()
wwlog.Printf(wwlog.INFO, "Building overlay for %s: host\n", hostname)
wwlog.Info("Building overlay for host: %s", hostname)
idEntry.Set(hostname)
host.Id = idEntry
return overlay.BuildOverlayIndir(host, strings.Split(OverlayName, ","), OverlayDir)
return overlay.BuildOverlayIndir(host, OverlayNames, OverlayDir)
}
}
if BuildHost || (!BuildHost && !BuildNodes && len(args) == 0 && controller.Warewulf.EnableHostOverlay) {
err := overlay.BuildHostOverlay()
if err != nil {
wwlog.Printf(wwlog.WARN, "host overlay could not be built: %s\n", err)
}
}
if BuildNodes || (!BuildHost && !BuildNodes) {
if len(args) > 0 {
args = hostlist.Expand(args)
if OverlayName != "" {
err = overlay.BuildSpecificOverlays(node.FilterByName(nodes, args), OverlayName)
} else {
err = overlay.BuildAllOverlays(node.FilterByName(nodes, args))
}
if BuildNodes || (!BuildHost && !BuildNodes) {
if len(OverlayNames) > 0 {
err = overlay.BuildSpecificOverlays(nodes, OverlayNames)
} else {
if OverlayName != "" {
for _, n := range nodes {
if util.InSlice(n.RuntimeOverlay.GetSlice(), OverlayName) ||
util.InSlice(n.SystemOverlay.GetSlice(), OverlayName) {
err = overlay.BuildSpecificOverlays([]node.NodeInfo{n}, OverlayName)
}
}
} else {
err = overlay.BuildAllOverlays(nodes)
}
err = overlay.BuildAllOverlays(nodes)
}
if err != nil {
wwlog.Printf(wwlog.WARN, "Some system overlays failed to be generated: %s\n", err)
wwlog.Printf(wwlog.WARN, "Some overlays failed to be generated: %s\n", err)
}
}
return nil

5
internal/app/wwctl/overlay/build/root.go
View File

@@ -31,14 +31,15 @@ var (
}
BuildHost bool
BuildNodes bool
OverlayName string
OverlayNames []string
OverlayDir string
)
func init() {
baseCmd.PersistentFlags().BoolVarP(&BuildHost, "host", "H", false, "Build overlays only for the host")
baseCmd.PersistentFlags().BoolVarP(&BuildNodes, "nodes", "N", false, "Build overlays only for the nodes")
baseCmd.PersistentFlags().StringVarP(&OverlayName, "overlay", "O", "", "Build only specific overlay")
baseCmd.PersistentFlags().StringSliceVarP(&OverlayNames, "overlay", "O", []string{}, "Build only specific overlay(s)")
if err := baseCmd.RegisterFlagCompletionFunc("overlay", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
list, _ := overlay.FindOverlays()
return list, cobra.ShellCompDirectiveNoFileComp

2
internal/app/wwctl/overlay/imprt/main.go
View File

@@ -70,7 +70,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
}
}
return overlay.BuildSpecificOverlays(updateNodes, overlayName)
return overlay.BuildSpecificOverlays(updateNodes, []string{overlayName})
}
return nil

8
internal/pkg/overlay/overlay.go
View File

@@ -68,13 +68,13 @@ func BuildAllOverlays(nodes []node.NodeInfo) error {
// TODO: Add an Overlay Delete for both sourcedir and image
func BuildSpecificOverlays(nodes []node.NodeInfo, overlayName string) error {
func BuildSpecificOverlays(nodes []node.NodeInfo, overlayNames []string) error {
for _, n := range nodes {
wwlog.Info("Building overlay for %s: %s", n.Id.Get(), overlayName)
err := BuildOverlay(n, []string{overlayName})
wwlog.Info("Building overlay for %s: %v", n.Id.Get(), overlayNames)
err := BuildOverlay(n, overlayNames)
if err != nil {
return errors.Wrap(err, "could not build overlay "+n.Id.Get()+"/"+overlayName+".img")
return errors.Wrapf(err, "could not build overlay for node %s: %v", n.Id.Get(), overlayNames)
}
}

61
internal/pkg/util/util.go
View File

@@ -12,7 +12,9 @@ import (
"path"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync/atomic"
"syscall"
"time"
@@ -20,6 +22,45 @@ import (
"github.com/pkg/errors"
)
// reserve some number of cpus for system/warwulfd usage
var processLimitedReserve int = 4
// maximum number of concurrent spawned processes
var processLimitedMax = MaxInt(1, runtime.NumCPU() - processLimitedReserve)
// Channel used as semaphore to specififed processLimitedMax
var processLimitedChan = make(chan int, processLimitedMax)
// Current number of processes started + queued
var processLimitedNum int32 = 0
// Counter over total history of started processes
var processLimitedCounter uint32 = 0
func ProcessLimitedEnter() (index uint32) {
atomic.AddInt32(&processLimitedNum, 1)
index = atomic.AddUint32(&processLimitedCounter, 1)
// NOTE: blocks when channel is full (i.e. processLimitedMax)
// until a process exists and takes one out of channel
processLimitedChan <- 1
return
}
func ProcessLimitedExit() {
<-processLimitedChan
atomic.AddInt32(&processLimitedNum, -1)
}
func ProcessLimitedStatus() (running int32, queued int32) {
running = int32(len(processLimitedChan))
queued = processLimitedNum - running
return
}
func MaxInt( a int, b int ) int {
if a > b {
return a
}
return b
}
func FirstError(errs ...error) (err error) {
for _, e := range errs {
if err == nil {
@@ -654,3 +695,23 @@ func BuildFsImage(
return nil
}
/*******************************************************************************
Runs wwctl command
*/
func RunWWCTL(args ...string) (out []byte, err error) {
index := ProcessLimitedEnter()
defer ProcessLimitedExit()
running, queued := ProcessLimitedStatus()
wwlog.Verbose("Starting wwctl process %d (%d running, %d queued): %v",
index, running, queued, args )
proc := exec.Command("wwctl", args...)
out, err = proc.CombinedOutput()
wwlog.Verbose("Finished wwctl process %d", index)
return out, err
}

79
internal/pkg/warewulfd/nodedb.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
)
type nodeDB struct {
@@ -19,6 +20,12 @@ var (
)
func LoadNodeDB() error {
db.lock.Lock()
defer db.lock.Unlock()
return loadNodeDB()
}
func loadNodeDB() error {
TmpMap := make(map[string]node.NodeInfo)
DB, err := node.New()
@@ -38,8 +45,6 @@ func LoadNodeDB() error {
}
}
db.lock.Lock()
defer db.lock.Unlock()
db.NodeInfo = TmpMap
return nil
@@ -48,6 +53,10 @@ func LoadNodeDB() error {
func GetNode(val string) (node.NodeInfo, error) {
db.lock.RLock()
defer db.lock.RUnlock()
return getNode(val)
}
func getNode(val string) (node.NodeInfo, error) {
if _, ok := db.NodeInfo[val]; ok {
@@ -57,3 +66,69 @@ func GetNode(val string) (node.NodeInfo, error) {
var empty node.NodeInfo
return empty, errors.New("No node found")
}
func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) {
db.lock.Lock()
defer db.lock.Unlock()
return getNodeOrSetDiscoverable(hwaddr)
}
func getNodeOrSetDiscoverable(hwaddr string) (node.NodeInfo, error) {
// NOTE: since discoverable nodes will write an updated DB to file and then
// reload, it is not enough to lock individual reads from the DB
// to ensure the condition on which the node is updated is still satisfied
// after the DB is read back in.
n, err := getNode(hwaddr)
if err == nil {
return n, nil
}
// If we failed to find a node, let's see if we can add one...
var netdev string
wwlog.WarnExc(err, "%s (node not configured)", hwaddr)
config, err := node.New()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to read node configuration file)", hwaddr)
}
_n, netdev, err := config.FindDiscoverableNode()
if err != nil {
// NOTE: this is taken as there is no discoverable node, so return the
// empty one
return n, nil
}
_n.NetDevs[netdev].Hwaddr.Set(hwaddr)
_n.Discoverable.SetB(false)
// NOTE: errors here should return the empty node if the state cannot
// be saved and re-loaded, since subsequent requests will be made on invalid
// assumption that the database is up to date.
err = config.NodeUpdate(_n)
if err != nil {
return n, errors.Wrapf(err, "%s (failed to set node configuration)", hwaddr)
}
err = config.Persist()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr)
}
err = loadNodeDB()
if err != nil {
return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr)
}
// NOTE: previously all overlays were built here, but that will also
// be done automatically when attempting to serve an overlay that
// hasn't been built (without blocking the database).
wwlog.Serv("%s (node automatically configured)", hwaddr)
// return the discovered node
return _n, nil
}

69
internal/pkg/warewulfd/provision.go
View File

@@ -1,17 +1,15 @@
package warewulfd
import (
"bytes"
"net/http"
"path"
"strconv"
"bytes"
"text/template"
"github.com/hpcng/warewulf/internal/pkg/buildconfig"
nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/container"
"github.com/hpcng/warewulf/internal/pkg/kernel"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/warewulfconf"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
@@ -70,47 +68,11 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
// TODO: when module version is upgraded to go1.18, should be 'any' type
var tmpl_data interface{}
node, err := GetNode(rinfo.hwaddr)
node, err := GetNodeOrSetDiscoverable(rinfo.hwaddr)
if err != nil {
// If we failed to find a node, let's see if we can add one...
var netdev string
wwlog.Warn("%s (node not configured)", rinfo.hwaddr)
nodeDB, err := nodepkg.New()
if err != nil {
wwlog.Error("Could not read node configuration file: %s", err)
w.WriteHeader(http.StatusServiceUnavailable)
return
}
n, netdev, err := nodeDB.FindDiscoverableNode()
if err == nil {
n.NetDevs[netdev].Hwaddr.Set(rinfo.hwaddr)
n.Discoverable.SetB(false)
err := nodeDB.NodeUpdate(n)
if err != nil {
wwlog.Serv("%s (failed to set node configuration)", rinfo.hwaddr)
} else {
err := nodeDB.Persist()
if err != nil {
wwlog.Serv("%s (failed to persist node configuration)", rinfo.hwaddr)
} else {
node = n
_ = overlay.BuildAllOverlays([]nodepkg.NodeInfo{n})
wwlog.Serv("%s (node automatically configured)", rinfo.hwaddr)
err := LoadNodeDB()
if err != nil {
wwlog.Warn("Could not reload configuration: %s", err)
}
}
}
}
wwlog.ErrorExc(err, "")
w.WriteHeader(http.StatusServiceUnavailable)
return
}
if node.AssetKey.Defined() && node.AssetKey.Get() != rinfo.assetkey {
@@ -188,16 +150,15 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
}
if len(stage_overlays) > 0 {
stage_file = overlay.OverlayImage(node.Id.Get(), stage_overlays)
if conf.Warewulf.AutobuildOverlays {
oneoverlaynewer := false
for _, overlayname := range stage_overlays {
oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
}
if !util.IsFile(stage_file) || util.PathIsNewer(stage_file, nodepkg.ConfigFile) || oneoverlaynewer {
wwlog.Serv("BUILD %15s, overlays %v", node.Id.Get(), stage_overlays)
_ = overlay.BuildOverlay(node, stage_overlays)
}
stage_file, err = getOverlayFile(
node.Id.Get(),
stage_overlays,
conf.Warewulf.AutobuildOverlays )
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
wwlog.ErrorExc(err, "")
return
}
}
@@ -255,7 +216,7 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusNotFound)
}
err = sendFile(w, stage_file, node.Id.Get())
err = sendFile(w, req, stage_file, node.Id.Get())
if err != nil {
wwlog.ErrorExc(err, "")
return

85
internal/pkg/warewulfd/util.go
View File

@@ -1,16 +1,21 @@
package warewulfd
import (
"io"
"net/http"
"os"
"strconv"
nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"github.com/pkg/errors"
)
func sendFile(w http.ResponseWriter, filename string, sendto string) error {
func sendFile(
w http.ResponseWriter,
req *http.Request,
filename string,
sendto string) error {
fd, err := os.Open(filename)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
@@ -18,34 +23,60 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) error {
}
defer fd.Close()
FileHeader := make([]byte, 512)
_, err = fd.Read(FileHeader)
stat, err := fd.Stat()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to read header")
return err
}
FileContentType := http.DetectContentType(FileHeader)
FileStat, _ := fd.Stat()
FileSize := strconv.FormatInt(FileStat.Size(), 10)
_, err = fd.Seek(0, 0)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to seek")
}
w.Header().Set("Content-Disposition", "attachment; filename=kernel")
w.Header().Set("Content-Type", FileContentType)
w.Header().Set("Content-Length", FileSize)
_, err = io.Copy(w, fd)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return errors.Wrap(err, "failed to copy")
}
http.ServeContent(
w,
req,
filename,
stat.ModTime(),
fd )
wwlog.Send("%15s: %s", sendto, filename)
return err
return nil
}
func getOverlayFile(
nodeId string,
stage_overlays []string,
autobuild bool ) (stage_file string, err error) {
stage_file = overlay.OverlayImage(nodeId, stage_overlays)
err = nil
build := !util.IsFile(stage_file)
if !build && autobuild {
build = util.PathIsNewer(stage_file, nodepkg.ConfigFile)
for _, overlayname := range stage_overlays {
build = build || util.PathIsNewer(stage_file, overlay.OverlaySourceDir(overlayname))
}
}
if build {
wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays)
args := []string{"overlay", "build"}
for _, overlayname := range stage_overlays {
args = append(args, "-O", overlayname)
}
args = append(args, nodeId)
out, err := util.RunWWCTL(args...)
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
nodeId, stage_overlays, stage_file, string(out))
}
}
return
}