Simplified version of node status monitoring with CLI

go.mod

@@ -12,5 +12,6 @@ require (
 	github.com/opencontainers/umoci v0.4.6
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.1.1
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -892,6 +892,7 @@ golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/app/wwctl/node/root.go

@@ -8,14 +8,15 @@ import (
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/ready"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/sensors"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/set"
+	nodestatus "github.com/hpcng/warewulf/internal/app/wwctl/node/status"
 	"github.com/spf13/cobra"
 )
 
 var (
 	baseCmd = &cobra.Command{
 		DisableFlagsInUseLine: true,
-		Use:   "node COMMAND [OPTONS]",
+		Use:                   "node COMMAND [OPTONS]",
-		Short: "Node management",
+		Short:                 "Node management",
 		Long: "Management of node settings. All node ranges can use brackets to identify\n" +
 			"node ranges. For example: n00[00-4].cluster[0-1] will identify the first 5 nodes\n" +
 			"in cluster0 and cluster1.",
@@ -30,6 +31,7 @@ func init() {
 	baseCmd.AddCommand(delete.GetCommand())
 	baseCmd.AddCommand(console.GetCommand())
 	baseCmd.AddCommand(ready.GetCommand())
+	baseCmd.AddCommand(nodestatus.GetCommand())
 }
 
 // GetRootCommand returns the root cobra.Command for the application.

internal/app/wwctl/node/status/main.go

@@ -0,0 +1,116 @@
+package nodestatus
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/warewulfconf"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/pkg/hostlist"
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+)
+
+type allStatus struct {
+	Nodes map[string]*NodeStatus `json:"nodes"`
+}
+
+type NodeStatus struct {
+	Stage    string `json:"stage"`
+	Sent     string `json:"sent"`
+	Ipaddr   string `json:"ipaddr"`
+	Lastseen int64  `json:"last seen"`
+}
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+
+	nodeDB, err := node.New()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err)
+		os.Exit(1)
+	}
+
+	nodes, err := nodeDB.FindAllNodes()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err)
+		os.Exit(1)
+	}
+
+	controller, err := warewulfconf.New()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "%s\n", err)
+		os.Exit(1)
+	}
+
+	if controller.Ipaddr == "" {
+		wwlog.Printf(wwlog.ERROR, "The Warewulf Server IP Address is not properly configured\n")
+		os.Exit(1)
+	}
+
+	statusURL := fmt.Sprintf("http://%s:%d/status", controller.Ipaddr, controller.Warewulf.Port)
+
+	for {
+		var height int
+		count := 4
+		rightnow := time.Now().Unix()
+
+		wwlog.Printf(wwlog.VERBOSE, "Connecting to: %s\n", statusURL)
+
+		resp, err := http.Get(statusURL)
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "Could not connect to Warewulf server: %s\n", err)
+			os.Exit(1)
+		}
+		defer resp.Body.Close()
+
+		decoder := json.NewDecoder(resp.Body)
+		var nodeStatus allStatus
+
+		err = decoder.Decode(&nodeStatus)
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "Could not decode JSON: %s\n", err)
+			os.Exit(1)
+		}
+
+		if SetWatch {
+			fmt.Print("\033[H\033[2J")
+			_, height, err = term.GetSize(0)
+			if err != nil {
+				wwlog.Printf(wwlog.WARN, "Could not get terminal height, using 24\n")
+				height = 24
+			}
+		}
+
+		args = hostlist.Expand(args)
+
+		fmt.Printf("%-20s %-20s %-25s %-10s\n", "NODENAME", "STATUS", "SENT", "LASTSEEN (s)")
+		fmt.Printf("%s\n", strings.Repeat("=", 80))
+
+		for _, node := range node.FilterByName(nodes, args) {
+			id := node.Id.Get()
+			if _, ok := nodeStatus.Nodes[id]; ok {
+				fmt.Printf("%-20s %-20s %-25s %-10d\n", id, nodeStatus.Nodes[id].Stage, nodeStatus.Nodes[id].Sent, rightnow-nodeStatus.Nodes[id].Lastseen)
+			} else {
+				fmt.Printf("%-20s %-20s %-25s %-10s\n", id, "--", "--", "--")
+			}
+			if count >= height && SetWatch {
+				break
+			}
+			count++
+		}
+
+		if SetWatch {
+			fmt.Printf("... ")
+			time.Sleep(1000 * time.Millisecond)
+		} else {
+			break
+		}
+	}
+
+	return nil
+}

internal/app/wwctl/node/status/root.go

@@ -0,0 +1,24 @@
+package nodestatus
+
+import "github.com/spf13/cobra"
+
+var (
+	baseCmd = &cobra.Command{
+		DisableFlagsInUseLine: true,
+		Use:                   "status [OPTIONS] [NODENAME...]",
+		Short:                 "View the provisioning status of nodes",
+		Long:                  "View and monitor the status of nodes as they are provisioned and check in.",
+		RunE:                  CobraRunE,
+		Args:                  cobra.MinimumNArgs(0),
+	}
+	SetWatch bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVar(&SetWatch, "watch", false, "Watch the status automatically")
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}

internal/pkg/warewulfd/container.go

@@ -2,6 +2,7 @@ package warewulfd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hpcng/warewulf/internal/pkg/container"
 )
@@ -22,6 +23,9 @@ func ContainerSend(w http.ResponseWriter, req *http.Request) {
 			daemonLogf("ERROR: %s\n", err)
 			w.WriteHeader(503)
 		}
+
+		updateStatus(node.Id.Get(), "CONTAINER", node.ContainerName.Get()+".img", strings.Split(req.RemoteAddr, ":")[0])
+
 	} else {
 		w.WriteHeader(503)
 		daemonLogf("WARNING: No Container set for node %s\n", node.Id.Get())

internal/pkg/warewulfd/ipxe.go

@@ -150,5 +150,6 @@ func IpxeSend(w http.ResponseWriter, req *http.Request) {
 
 		daemonLogf("SEND:  %15s: %s\n", nodeobj.Id.Get(), ipxeTemplate)
 
+		updateStatus(nodeobj.Id.Get(), "iPXE", nodeobj.Ipxe.Get()+".ipxe", strings.Split(req.RemoteAddr, ":")[0])
 	}
 }

internal/pkg/warewulfd/kernel.go

@@ -2,6 +2,7 @@ package warewulfd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hpcng/warewulf/internal/pkg/kernel"
 )
@@ -22,6 +23,8 @@ func KernelSend(w http.ResponseWriter, req *http.Request) {
 			daemonLogf("ERROR: %s\n", err)
 		}
 
+		updateStatus(node.Id.Get(), "KERNEL", node.KernelVersion.Get(), strings.Split(req.RemoteAddr, ":")[0])
+
 	} else {
 		w.WriteHeader(503)
 		daemonLogf("WARNING: No 'kernel version' set for node %s\n", node.Id.Get())

internal/pkg/warewulfd/kmods.go

@@ -2,6 +2,7 @@ package warewulfd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hpcng/warewulf/internal/pkg/kernel"
 )
@@ -23,6 +24,8 @@ func KmodsSend(w http.ResponseWriter, req *http.Request) {
 			daemonLogf("ERROR: %s\n", err)
 		}
 
+		updateStatus(node.Id.Get(), "KMODS", node.KernelVersion.Get()+".img", strings.Split(req.RemoteAddr, ":")[0])
+
 	} else {
 		w.WriteHeader(503)
 		daemonLogf("WARNING: No 'kernel version' set for node %s\n", node.Id.Get())

internal/pkg/warewulfd/runtime.go

@@ -77,6 +77,9 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) {
 		if err != nil {
 			daemonLogf("ERROR: %s\n", err)
 		}
+
+		updateStatus(n.Id.Get(), "RUNTIME OVERLAY", n.RuntimeOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0])
+
 	} else {
 		w.WriteHeader(503)
 		daemonLogf("WARNING: No 'runtime system-overlay' set for node %s\n", n.Id.Get())

internal/pkg/warewulfd/status.go

@@ -0,0 +1,64 @@
+package warewulfd
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/pkg/errors"
+)
+
+type allStatus struct {
+	Nodes map[string]*NodeStatus `json:"nodes"`
+}
+
+type NodeStatus struct {
+	Stage    string `json:"stage"`
+	Sent     string `json:"sent"`
+	Ipaddr   string `json:"ipaddr"`
+	Lastseen int64  `json:"last seen"`
+}
+
+var statusDB allStatus
+
+func init() {
+	statusDB.Nodes = make(map[string]*NodeStatus)
+}
+
+func updateStatus(nodeID, stage, sent, ipaddr string) error {
+	rightnow := time.Now().Unix()
+
+	wwlog.Printf(wwlog.DEBUG, "Updating node status data: %s\n", nodeID)
+
+	var n NodeStatus
+	n.Stage = stage
+	n.Lastseen = rightnow
+	n.Sent = sent
+	n.Ipaddr = ipaddr
+	statusDB.Nodes[nodeID] = &n
+
+	return nil
+}
+
+func statusJSON() ([]byte, error) {
+
+	wwlog.Printf(wwlog.DEBUG, "Request for node status data...\n")
+
+	ret, err := json.MarshalIndent(statusDB, "", "  ")
+	if err != nil {
+		return ret, errors.Wrap(err, "could not marshal JSON data from sstatus structure")
+	}
+
+	return ret, nil
+}
+
+func StatusSend(w http.ResponseWriter, req *http.Request) {
+
+	status, err := statusJSON()
+	if err != nil {
+		return
+	}
+
+	w.Write(status)
+}

internal/pkg/warewulfd/system.go

@@ -2,6 +2,7 @@ package warewulfd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hpcng/warewulf/internal/pkg/config"
 	"github.com/hpcng/warewulf/internal/pkg/node"
@@ -39,6 +40,9 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) {
 		if err != nil {
 			daemonLogf("ERROR: %s\n", err)
 		}
+
+		updateStatus(n.Id.Get(), "SYSTEM OVERLAY", n.SystemOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0])
+
 	} else {
 		w.WriteHeader(503)
 		daemonLogf("WARNING: No 'system system-overlay' set for node %s\n", n.Id.Get())

internal/pkg/warewulfd/warewulfd.go

@@ -40,6 +40,7 @@ func RunServer() error {
 	http.HandleFunc("/container/", ContainerSend)
 	http.HandleFunc("/overlay-system/", SystemOverlaySend)
 	http.HandleFunc("/overlay-runtime/", RuntimeOverlaySend)
+	http.HandleFunc("/status", StatusSend)
 
 	conf, err := warewulfconf.New()
 	if err != nil {