From 525ac1184ecdea33b682630417ae40b1f4c49bec Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Dec 2021 23:26:45 +0000 Subject: [PATCH] Simplified version of node status monitoring with CLI --- go.mod | 1 + go.sum | 1 + internal/app/wwctl/node/root.go | 6 +- internal/app/wwctl/node/status/main.go | 116 +++++++++++++++++++++++++ internal/app/wwctl/node/status/root.go | 24 +++++ internal/pkg/warewulfd/container.go | 4 + internal/pkg/warewulfd/ipxe.go | 1 + internal/pkg/warewulfd/kernel.go | 3 + internal/pkg/warewulfd/kmods.go | 3 + internal/pkg/warewulfd/runtime.go | 3 + internal/pkg/warewulfd/status.go | 64 ++++++++++++++ internal/pkg/warewulfd/system.go | 4 + internal/pkg/warewulfd/warewulfd.go | 1 + 13 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 internal/app/wwctl/node/status/main.go create mode 100644 internal/app/wwctl/node/status/root.go create mode 100644 internal/pkg/warewulfd/status.go diff --git a/go.mod b/go.mod index dc18e0cb..34a0c2c5 100644 --- a/go.mod +++ b/go.mod @@ -12,5 +12,6 @@ require ( github.com/opencontainers/umoci v0.4.6 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.1.1 + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 3ae31d68..4dceb5bf 100644 --- a/go.sum +++ b/go.sum @@ -892,6 +892,7 @@ golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/app/wwctl/node/root.go b/internal/app/wwctl/node/root.go index a8b10061..dd165138 100644 --- a/internal/app/wwctl/node/root.go +++ b/internal/app/wwctl/node/root.go @@ -8,14 +8,15 @@ import ( "github.com/hpcng/warewulf/internal/app/wwctl/node/ready" "github.com/hpcng/warewulf/internal/app/wwctl/node/sensors" "github.com/hpcng/warewulf/internal/app/wwctl/node/set" + nodestatus "github.com/hpcng/warewulf/internal/app/wwctl/node/status" "github.com/spf13/cobra" ) var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "node COMMAND [OPTONS]", - Short: "Node management", + Use: "node COMMAND [OPTONS]", + Short: "Node management", Long: "Management of node settings. All node ranges can use brackets to identify\n" + "node ranges. For example: n00[00-4].cluster[0-1] will identify the first 5 nodes\n" + "in cluster0 and cluster1.", @@ -30,6 +31,7 @@ func init() { baseCmd.AddCommand(delete.GetCommand()) baseCmd.AddCommand(console.GetCommand()) baseCmd.AddCommand(ready.GetCommand()) + baseCmd.AddCommand(nodestatus.GetCommand()) } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/node/status/main.go b/internal/app/wwctl/node/status/main.go new file mode 100644 index 00000000..de201521 --- /dev/null +++ b/internal/app/wwctl/node/status/main.go @@ -0,0 +1,116 @@ +package nodestatus + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/warewulfconf" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/pkg/hostlist" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +type allStatus struct { + Nodes map[string]*NodeStatus `json:"nodes"` +} + +type NodeStatus struct { + Stage string `json:"stage"` + Sent string `json:"sent"` + Ipaddr string `json:"ipaddr"` + Lastseen int64 `json:"last seen"` +} + +func CobraRunE(cmd *cobra.Command, args []string) error { + + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + nodes, err := nodeDB.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) + os.Exit(1) + } + + controller, err := warewulfconf.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + + if controller.Ipaddr == "" { + wwlog.Printf(wwlog.ERROR, "The Warewulf Server IP Address is not properly configured\n") + os.Exit(1) + } + + statusURL := fmt.Sprintf("http://%s:%d/status", controller.Ipaddr, controller.Warewulf.Port) + + for { + var height int + count := 4 + rightnow := time.Now().Unix() + + wwlog.Printf(wwlog.VERBOSE, "Connecting to: %s\n", statusURL) + + resp, err := http.Get(statusURL) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not connect to Warewulf server: %s\n", err) + os.Exit(1) + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + var nodeStatus allStatus + + err = decoder.Decode(&nodeStatus) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not decode JSON: %s\n", err) + os.Exit(1) + } + + if SetWatch { + fmt.Print("\033[H\033[2J") + _, height, err = term.GetSize(0) + if err != nil { + wwlog.Printf(wwlog.WARN, "Could not get terminal height, using 24\n") + height = 24 + } + } + + args = hostlist.Expand(args) + + fmt.Printf("%-20s %-20s %-25s %-10s\n", "NODENAME", "STATUS", "SENT", "LASTSEEN (s)") + fmt.Printf("%s\n", strings.Repeat("=", 80)) + + for _, node := range node.FilterByName(nodes, args) { + id := node.Id.Get() + if _, ok := nodeStatus.Nodes[id]; ok { + fmt.Printf("%-20s %-20s %-25s %-10d\n", id, nodeStatus.Nodes[id].Stage, nodeStatus.Nodes[id].Sent, rightnow-nodeStatus.Nodes[id].Lastseen) + } else { + fmt.Printf("%-20s %-20s %-25s %-10s\n", id, "--", "--", "--") + } + if count >= height && SetWatch { + break + } + count++ + } + + if SetWatch { + fmt.Printf("... ") + time.Sleep(1000 * time.Millisecond) + } else { + break + } + } + + return nil +} diff --git a/internal/app/wwctl/node/status/root.go b/internal/app/wwctl/node/status/root.go new file mode 100644 index 00000000..3423bc48 --- /dev/null +++ b/internal/app/wwctl/node/status/root.go @@ -0,0 +1,24 @@ +package nodestatus + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + DisableFlagsInUseLine: true, + Use: "status [OPTIONS] [NODENAME...]", + Short: "View the provisioning status of nodes", + Long: "View and monitor the status of nodes as they are provisioned and check in.", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(0), + } + SetWatch bool +) + +func init() { + baseCmd.PersistentFlags().BoolVar(&SetWatch, "watch", false, "Watch the status automatically") +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/pkg/warewulfd/container.go b/internal/pkg/warewulfd/container.go index 355f62ee..5eff5dc0 100644 --- a/internal/pkg/warewulfd/container.go +++ b/internal/pkg/warewulfd/container.go @@ -2,6 +2,7 @@ package warewulfd import ( "net/http" + "strings" "github.com/hpcng/warewulf/internal/pkg/container" ) @@ -22,6 +23,9 @@ func ContainerSend(w http.ResponseWriter, req *http.Request) { daemonLogf("ERROR: %s\n", err) w.WriteHeader(503) } + + updateStatus(node.Id.Get(), "CONTAINER", node.ContainerName.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + } else { w.WriteHeader(503) daemonLogf("WARNING: No Container set for node %s\n", node.Id.Get()) diff --git a/internal/pkg/warewulfd/ipxe.go b/internal/pkg/warewulfd/ipxe.go index 2eec83ec..d60fd4c3 100644 --- a/internal/pkg/warewulfd/ipxe.go +++ b/internal/pkg/warewulfd/ipxe.go @@ -150,5 +150,6 @@ func IpxeSend(w http.ResponseWriter, req *http.Request) { daemonLogf("SEND: %15s: %s\n", nodeobj.Id.Get(), ipxeTemplate) + updateStatus(nodeobj.Id.Get(), "iPXE", nodeobj.Ipxe.Get()+".ipxe", strings.Split(req.RemoteAddr, ":")[0]) } } diff --git a/internal/pkg/warewulfd/kernel.go b/internal/pkg/warewulfd/kernel.go index 58196a72..73f54369 100644 --- a/internal/pkg/warewulfd/kernel.go +++ b/internal/pkg/warewulfd/kernel.go @@ -2,6 +2,7 @@ package warewulfd import ( "net/http" + "strings" "github.com/hpcng/warewulf/internal/pkg/kernel" ) @@ -22,6 +23,8 @@ func KernelSend(w http.ResponseWriter, req *http.Request) { daemonLogf("ERROR: %s\n", err) } + updateStatus(node.Id.Get(), "KERNEL", node.KernelVersion.Get(), strings.Split(req.RemoteAddr, ":")[0]) + } else { w.WriteHeader(503) daemonLogf("WARNING: No 'kernel version' set for node %s\n", node.Id.Get()) diff --git a/internal/pkg/warewulfd/kmods.go b/internal/pkg/warewulfd/kmods.go index 473f8e77..03564f93 100644 --- a/internal/pkg/warewulfd/kmods.go +++ b/internal/pkg/warewulfd/kmods.go @@ -2,6 +2,7 @@ package warewulfd import ( "net/http" + "strings" "github.com/hpcng/warewulf/internal/pkg/kernel" ) @@ -23,6 +24,8 @@ func KmodsSend(w http.ResponseWriter, req *http.Request) { daemonLogf("ERROR: %s\n", err) } + updateStatus(node.Id.Get(), "KMODS", node.KernelVersion.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + } else { w.WriteHeader(503) daemonLogf("WARNING: No 'kernel version' set for node %s\n", node.Id.Get()) diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index eaf17eef..ec63ab63 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -77,6 +77,9 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { if err != nil { daemonLogf("ERROR: %s\n", err) } + + updateStatus(n.Id.Get(), "RUNTIME OVERLAY", n.RuntimeOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + } else { w.WriteHeader(503) daemonLogf("WARNING: No 'runtime system-overlay' set for node %s\n", n.Id.Get()) diff --git a/internal/pkg/warewulfd/status.go b/internal/pkg/warewulfd/status.go new file mode 100644 index 00000000..2fc19301 --- /dev/null +++ b/internal/pkg/warewulfd/status.go @@ -0,0 +1,64 @@ +package warewulfd + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/pkg/errors" +) + +type allStatus struct { + Nodes map[string]*NodeStatus `json:"nodes"` +} + +type NodeStatus struct { + Stage string `json:"stage"` + Sent string `json:"sent"` + Ipaddr string `json:"ipaddr"` + Lastseen int64 `json:"last seen"` +} + +var statusDB allStatus + +func init() { + statusDB.Nodes = make(map[string]*NodeStatus) +} + +func updateStatus(nodeID, stage, sent, ipaddr string) error { + rightnow := time.Now().Unix() + + wwlog.Printf(wwlog.DEBUG, "Updating node status data: %s\n", nodeID) + + var n NodeStatus + n.Stage = stage + n.Lastseen = rightnow + n.Sent = sent + n.Ipaddr = ipaddr + statusDB.Nodes[nodeID] = &n + + return nil +} + +func statusJSON() ([]byte, error) { + + wwlog.Printf(wwlog.DEBUG, "Request for node status data...\n") + + ret, err := json.MarshalIndent(statusDB, "", " ") + if err != nil { + return ret, errors.Wrap(err, "could not marshal JSON data from sstatus structure") + } + + return ret, nil +} + +func StatusSend(w http.ResponseWriter, req *http.Request) { + + status, err := statusJSON() + if err != nil { + return + } + + w.Write(status) +} diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index f63fdac1..0500b8ec 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -2,6 +2,7 @@ package warewulfd import ( "net/http" + "strings" "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/node" @@ -39,6 +40,9 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { if err != nil { daemonLogf("ERROR: %s\n", err) } + + updateStatus(n.Id.Get(), "SYSTEM OVERLAY", n.SystemOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + } else { w.WriteHeader(503) daemonLogf("WARNING: No 'system system-overlay' set for node %s\n", n.Id.Get()) diff --git a/internal/pkg/warewulfd/warewulfd.go b/internal/pkg/warewulfd/warewulfd.go index 2776590b..01baa656 100644 --- a/internal/pkg/warewulfd/warewulfd.go +++ b/internal/pkg/warewulfd/warewulfd.go @@ -40,6 +40,7 @@ func RunServer() error { http.HandleFunc("/container/", ContainerSend) http.HandleFunc("/overlay-system/", SystemOverlaySend) http.HandleFunc("/overlay-runtime/", RuntimeOverlaySend) + http.HandleFunc("/status", StatusSend) conf, err := warewulfconf.New() if err != nil {