Added parallel batching, IPMI power cycle, sensors, and console

internal/app/wwctl/node/console/power.go

@@ -0,0 +1,60 @@
+package console
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/power"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/spf13/cobra"
+	"os"
+)
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	var returnErr error = nil
+	var nodeList []node.NodeInfo
+
+	n, err := node.New()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err)
+		os.Exit(1)
+	}
+
+	if len(args) == 1 {
+		nodeList, _ = n.SearchByName(args[0])
+	} else {
+		wwlog.Printf(wwlog.ERROR, "Please specify a node\n")
+		os.Exit(255)
+	}
+
+	if len(nodeList) == 0 {
+		wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0])
+		os.Exit(255)
+	}
+
+	for _, node := range nodeList {
+
+		if node.IpmiIpaddr.Get() == "" {
+			wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get())
+			continue
+		}
+
+		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
+			HostName: node.IpmiIpaddr.Get(),
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
+			AuthType: "MD5",
+		}
+
+		err := ipmiCmd.Console()
+
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "%s: Console problem\n", node.Id.Get())
+			returnErr = err
+			continue
+		}
+
+	}
+
+	return returnErr
+}
+

internal/app/wwctl/node/console/root.go

@@ -0,0 +1,19 @@
+package console
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	powerCmd = &cobra.Command{
+		Use:   "console",
+		Short: "IPMI console",
+		Long:  "Start IPMI console for a singe node",
+		RunE:  CobraRunE,
+	}
+)
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return powerCmd
+}

internal/app/wwctl/node/powercycle/power.go

@@ -0,0 +1,81 @@
+package powercycle
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/power"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/internal/pkg/batch"
+	"github.com/spf13/cobra"
+	"os"
+	"fmt"
+)
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	var returnErr error = nil
+	var nodeList []node.NodeInfo
+
+	n, err := node.New()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err)
+		os.Exit(1)
+	}
+
+	if len(args) >= 1 {
+		nodeList, _ = n.SearchByNameList(args)
+	} else {
+		wwlog.Printf(wwlog.ERROR, "No requested nodes\n")
+		os.Exit(255)
+	}
+
+	if len(nodeList) == 0 {
+		wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0])
+		os.Exit(255)
+	}
+
+	batchpool := batch.New(50, 0)
+	jobcount := len(nodeList)
+	results := make(chan power.IPMI, jobcount)
+
+	for _, node := range nodeList {
+
+		if node.IpmiIpaddr.Get() == "" {
+			wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get())
+			continue
+		}
+
+		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
+			HostName: node.IpmiIpaddr.Get(),
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
+			AuthType: "MD5",
+		}
+
+		batchpool.Submit(func() {
+			ipmiCmd.PowerCycle()
+			results <- ipmiCmd
+		})
+
+	}
+
+	batchpool.Run()
+
+	close(results)
+
+	for result := range results {
+
+		out, err := result.Result()
+
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out)
+			returnErr = err
+			continue
+		}
+
+		fmt.Printf("%s: %s\n", result.NodeName, out)
+
+	}
+
+	return returnErr
+}
+

internal/app/wwctl/node/powercycle/root.go

@@ -0,0 +1,19 @@
+package powercycle
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	powerCmd = &cobra.Command{
+		Use:   "powercycle",
+		Short: "power cycle node(s)",
+		Long:  "cycle power for one or more nodes",
+		RunE:  CobraRunE,
+	}
+)
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return powerCmd
+}

internal/app/wwctl/node/poweroff/power.go

@@ -4,8 +4,10 @@ import (
 	"github.com/hpcng/warewulf/internal/pkg/node"
 	"github.com/hpcng/warewulf/internal/pkg/power"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/internal/pkg/batch"
 	"github.com/spf13/cobra"
 	"os"
+	"fmt"
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
@@ -28,10 +30,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	if len(nodeList) == 0 {
 		wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0])
 		os.Exit(255)
-	} else {
-		wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList))
 	}
 
+	batchpool := batch.New(50, 0)
+	jobcount := len(nodeList)
+	results := make(chan power.IPMI, jobcount)
+
 	for _, node := range nodeList {
 
 		if node.IpmiIpaddr.Get() == "" {
@@ -40,22 +44,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		}
 
 		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
 			HostName: node.IpmiIpaddr.Get(),
-			User:     "ADMIN",
-			Password: "ADMIN",
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
 			AuthType: "MD5",
 		}
 
-		out, err := ipmiCmd.PowerOff()
+		batchpool.Submit(func() {
+			ipmiCmd.PowerOff()
+			results <- ipmiCmd
+		})
+
+	}
+
+	batchpool.Run()
+
+	close(results)
+
+	for result := range results {
+
+		out, err := result.Result()
 
 		if err != nil {
-			wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out)
+			wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out)
 			returnErr = err
 			continue
 		}
 
-		wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out)
+		fmt.Printf("%s: %s\n", result.NodeName, out)
+
 	}
 
 	return returnErr
 }
+

internal/app/wwctl/node/poweron/power.go

@@ -4,8 +4,10 @@ import (
 	"github.com/hpcng/warewulf/internal/pkg/node"
 	"github.com/hpcng/warewulf/internal/pkg/power"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/internal/pkg/batch"
 	"github.com/spf13/cobra"
 	"os"
+	"fmt"
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
@@ -32,9 +34,11 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList))
 	}
 
-	for _, node := range nodeList {
+	batchpool := batch.New(50, 0)
+	jobcount := len(nodeList)
+	results := make(chan power.IPMI, jobcount)
 
-		var powerCmd power.PowerOnInterface
+	for _, node := range nodeList {
 
 		if node.IpmiIpaddr.Get() == "" {
 			wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get())
@@ -42,24 +46,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		}
 
 		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
 			HostName: node.IpmiIpaddr.Get(),
-			User:     "ADMIN",
-			Password: "ADMIN",
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
 			AuthType: "MD5",
 		}
 
-		powerCmd = ipmiCmd
+		batchpool.Submit(func() {
+			ipmiCmd.PowerOn()
+			results <- ipmiCmd
+		})
 
-		out, err := powerCmd.PowerOn()
+	}
+
+	batchpool.Run()
+
+	close(results)
+
+	for result := range results {
+
+		out, err := result.Result()
 
 		if err != nil {
-			wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out)
+			wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out)
 			returnErr = err
 			continue
 		}
 
-		wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out)
+		fmt.Printf("%s: %s\n", result.NodeName, out)
+
 	}
 
 	return returnErr
 }
+

internal/app/wwctl/node/powerstatus/power.go

@@ -4,8 +4,10 @@ import (
 	"github.com/hpcng/warewulf/internal/pkg/node"
 	"github.com/hpcng/warewulf/internal/pkg/power"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/internal/pkg/batch"
 	"github.com/spf13/cobra"
 	"os"
+	"fmt"
 )
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
@@ -32,6 +34,10 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList))
 	}
 
+	batchpool := batch.New(50, 0)
+	jobcount := len(nodeList)
+	results := make(chan power.IPMI, jobcount)
+
 	for _, node := range nodeList {
 
 		if node.IpmiIpaddr.Get() == "" {
@@ -40,22 +46,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 		}
 
 		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
 			HostName: node.IpmiIpaddr.Get(),
-			User:     "ADMIN",
-			Password: "ADMIN",
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
 			AuthType: "MD5",
 		}
 
-		out, err := ipmiCmd.PowerStatus()
+		batchpool.Submit(func() {
+			ipmiCmd.PowerStatus()
+			results <- ipmiCmd
+		})
+
+	}
+
+	batchpool.Run()
+
+	close(results)
+
+	for result := range results {
+
+		out, err := result.Result()
 
 		if err != nil {
-			wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out)
+			wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out)
 			returnErr = err
 			continue
 		}
 
-		wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out)
+		fmt.Printf("%s: %s\n", result.NodeName, out)
+
 	}
 
 	return returnErr
 }
+

internal/app/wwctl/node/root.go

@@ -6,7 +6,10 @@ import (
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/list"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/poweron"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/poweroff"
+	"github.com/hpcng/warewulf/internal/app/wwctl/node/powercycle"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/powerstatus"
+	"github.com/hpcng/warewulf/internal/app/wwctl/node/sensors"
+	"github.com/hpcng/warewulf/internal/app/wwctl/node/console"
 	"github.com/hpcng/warewulf/internal/app/wwctl/node/set"
 	"github.com/spf13/cobra"
 )
@@ -22,11 +25,14 @@ var (
 func init() {
 	baseCmd.AddCommand(poweron.GetCommand())
 	baseCmd.AddCommand(poweroff.GetCommand())
+	baseCmd.AddCommand(powercycle.GetCommand())
 	baseCmd.AddCommand(powerstatus.GetCommand())
+	baseCmd.AddCommand(sensors.GetCommand())
 	baseCmd.AddCommand(list.GetCommand())
 	baseCmd.AddCommand(set.GetCommand())
 	baseCmd.AddCommand(add.GetCommand())
 	baseCmd.AddCommand(delete.GetCommand())
+	baseCmd.AddCommand(console.GetCommand())
 }
 
 // GetRootCommand returns the root cobra.Command for the application.

internal/app/wwctl/node/sensors/power.go

@@ -0,0 +1,87 @@
+package sensors
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/hpcng/warewulf/internal/pkg/power"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/hpcng/warewulf/internal/pkg/batch"
+	"github.com/spf13/cobra"
+	"os"
+	"fmt"
+)
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	var returnErr error = nil
+	var nodeList []node.NodeInfo
+
+	n, err := node.New()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err)
+		os.Exit(1)
+	}
+
+	if len(args) >= 1 {
+		nodeList, _ = n.SearchByNameList(args)
+	} else {
+		wwlog.Printf(wwlog.ERROR, "No requested nodes\n")
+		os.Exit(255)
+	}
+
+	if len(nodeList) == 0 {
+		wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0])
+		os.Exit(255)
+	}
+
+	batchpool := batch.New(50, 0)
+	jobcount := len(nodeList)
+	results := make(chan power.IPMI, jobcount)
+
+	for _, node := range nodeList {
+
+		if node.IpmiIpaddr.Get() == "" {
+			wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get())
+			continue
+		}
+
+		ipmiCmd := power.IPMI{
+			NodeName: node.Id.Get(),
+			HostName: node.IpmiIpaddr.Get(),
+			User:     node.IpmiUserName.Get(),
+			Password: node.IpmiPassword.Get(),
+			AuthType: "MD5",
+		}
+
+		fullFlag := full
+
+		batchpool.Submit(func() {
+			if fullFlag == true {
+				ipmiCmd.SensorList()
+			} else {
+				ipmiCmd.SDRList()
+			}
+			results <- ipmiCmd
+		})
+
+	}
+
+	batchpool.Run()
+
+	close(results)
+
+	for result := range results {
+
+		out, err := result.Result()
+
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out)
+			returnErr = err
+			continue
+		}
+
+		fmt.Printf("%s:\n%s\n", result.NodeName, out)
+
+	}
+
+	return returnErr
+}
+

internal/app/wwctl/node/sensors/root.go

@@ -0,0 +1,24 @@
+package sensors
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	powerCmd = &cobra.Command{
+		Use:   "sensors",
+		Short: "node sensor information",
+		Long:  "Get Node Sensor Information",
+		RunE:  CobraRunE,
+	}
+	full bool
+)
+
+func init() {
+        powerCmd.PersistentFlags().BoolVarP(&full, "full", "F", false, "show detailed output.")
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return powerCmd
+}

internal/pkg/batch/batch.go

@@ -0,0 +1,57 @@
+
+package batch
+
+import (
+	"time"
+	"sync"
+)
+
+type BatchPool struct {
+	active int
+	mintime	int
+	jobcount int
+	jobs []func()
+}
+
+func New(active int, mintime int) *BatchPool {
+	pool := &BatchPool{
+		active: active,
+		mintime: mintime,
+	}
+	pool.jobs = []func(){}
+	return pool
+}
+
+func Min(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func (pool *BatchPool) Submit(f func()) {
+	pool.jobcount++
+	pool.jobs = append(pool.jobs, f)
+}
+
+func wrapper(wg *sync.WaitGroup, f func()) {
+	defer wg.Done()
+	f()
+}
+
+func (pool *BatchPool) Run() {
+	var wg sync.WaitGroup
+	count := pool.jobcount
+	for count > 0 {
+		for i:=0; i<Min(count, pool.active); i++ {
+			wg.Add(1)
+			go wrapper(&wg, pool.jobs[pool.jobcount-count])
+			count--
+		}
+		if (pool.mintime > 0) {
+			time.Sleep(time.Second * time.Duration(pool.mintime))
+		}
+		wg.Wait()
+	}
+}
+

internal/pkg/power/ipmitool.go

@@ -2,16 +2,28 @@ package power
 
 import (
 	"os/exec"
+	"os"
 )
 
+type IPMIResult struct {
+	err error
+	out string
+}
+
 type IPMI struct {
+	NodeName string
 	HostName string
 	User     string
 	Password string
 	AuthType string
+	result IPMIResult
 }
 
-func (ipmi IPMI) Command(ipmiArgs []string) ([]byte, error) {
+func (ipmi *IPMI) Result() (string, error) {
+	return ipmi.result.out, ipmi.result.err
+}
+
+func (ipmi *IPMI) Command(ipmiArgs []string) ([]byte, error) {
 
 	var args []string
 
@@ -21,29 +33,67 @@ func (ipmi IPMI) Command(ipmiArgs []string) ([]byte, error) {
 	return ipmiCmd.CombinedOutput()
 }
 
-func (ipmi IPMI) PowerOn() (string, error) {
+func (ipmi *IPMI) InteractiveCommand(ipmiArgs []string) error {
 
 	var args []string
 
-	args = append(args, "chassis", "power", "on")
-	ipmiOut, err := ipmi.Command(args)
-	return string(ipmiOut), err
+	args = append(args, "-I", "lanplus", "-H", ipmi.HostName, "-U", ipmi.User, "-P", ipmi.Password)
+	args = append(args, ipmiArgs...)
+	ipmiCmd := exec.Command("/usr/bin/ipmitool", args...)
+	ipmiCmd.Stdout = os.Stdout
+	ipmiCmd.Stdin = os.Stdin
+	ipmiCmd.Stderr = os.Stderr
+	return ipmiCmd.Run()
 }
 
-func (ipmi IPMI) PowerOff() (string, error) {
-
-	var args []string
-
-	args = append(args, "chassis", "power", "off")
-	ipmiOut, err := ipmi.Command(args)
-	return string(ipmiOut), err
+func (ipmi *IPMI) IPMIInteractiveCommand(args ...string) error {
+	return ipmi.InteractiveCommand(args)
 }
 
-func (ipmi IPMI) PowerStatus() (string, error) {
-
-	var args []string
-
-	args = append(args, "chassis", "power", "status")
+func (ipmi *IPMI) IPMICommand(args ...string) (string, error) {
 	ipmiOut, err := ipmi.Command(args)
-	return string(ipmiOut), err
+	ipmi.result.out = string(ipmiOut)
+	ipmi.result.err = err
+	return ipmi.result.out, ipmi.result.err
+
 }
+
+//func (ipmi IPMI) PowerOn() (string, error) {
+//
+//	var args []string
+//
+//	args = append(args, "chassis", "power", "on")
+//	ipmiOut, err := ipmi.Command(args)
+//	ipmi.result.out = string(ipmiOut)
+//	ipmi.result.err = err
+//	return ipmi.result.out, ipmi.result.err
+//}
+
+func (ipmi *IPMI) PowerOn() (string, error) {
+	return ipmi.IPMICommand("chassis", "power", "on")
+}
+
+func (ipmi *IPMI) PowerOff() (string, error) {
+	return ipmi.IPMICommand("chassis", "power", "off")
+}
+
+func (ipmi *IPMI) PowerCycle() (string, error) {
+	return ipmi.IPMICommand("chassis", "power", "cycle")
+}
+
+func (ipmi *IPMI) PowerStatus() (string, error) {
+	return ipmi.IPMICommand("chassis", "power", "status")
+}
+
+func (ipmi *IPMI) SDRList() (string, error) {
+	return ipmi.IPMICommand("sdr", "list")
+}
+
+func (ipmi *IPMI) SensorList() (string, error) {
+	return ipmi.IPMICommand("sensor", "list")
+}
+
+func (ipmi *IPMI) Console() error {
+	return ipmi.IPMIInteractiveCommand("sol", "activate")
+}
+

overlays/system/default/ipmi.ww

@@ -84,11 +84,14 @@ fi
 #ipmitool user priv 1 4 1
 
 # Serial Over LAN
+ipmitool channel setaccess 1 2 link=on ipmi=on callin=on privilege=4
 ipmitool sol set force-encryption true 1
 ipmitool sol set force-authentication true 1
 ipmitool sol set privilege-level admin 1
-ipmitool sol payload enable 1 4
-ipmitool sol set enabled true 1
-ipmitool sol set non-volatile-bit-rate 19.2 1
-ipmitool sol set volatile-bit-rate 19.2 1
+#ipmitool sol payload enable 1 4
+ipmitool sol payload enable 1 2 1
+ipmitool sol set enabled true 1 1
+speed=38.4  # 19.2 38.4 115.2
+ipmitool sol set non-volatile-bit-rate $speed 1
+ipmitool sol set volatile-bit-rate $speed 1
 

test/batch-test.go

@@ -0,0 +1,51 @@
+
+package main
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/batch"
+	"fmt"
+	"time"
+	"runtime"
+)
+
+func exampleJob(name string, id int) string {
+	fmt.Printf("[%d] %s started...\n", id, name)
+	time.Sleep(time.Second * 5)
+	fmt.Printf("[%d] %s ending...\n", id, name)
+	return name
+}
+
+
+func main() {
+
+	fmt.Printf("GOMAXPROCS=%d\n", runtime.GOMAXPROCS(0))
+
+	jobstorun := 1000
+
+	// create batch pool to run 100 jobs at a time every 10 seconds
+	batchpool := batch.New(100, 10)
+	retvalues := make(chan string, jobstorun)
+
+	// submit a bunch of jobs
+	for i:=0; i<jobstorun; i++ {
+
+		name := fmt.Sprintf("job-%04d", i)
+		id := i;
+
+		batchpool.Submit(func() {
+			retvalues <- exampleJob(name, id)
+		})
+
+	}
+
+	// Run all batch jobs to completion
+	batchpool.Run()
+
+	close(retvalues)
+
+	for s:= range retvalues {
+		fmt.Printf("Return value is %s\n", s)
+	}
+
+}
+