diff --git a/internal/app/wwctl/node/console/power.go b/internal/app/wwctl/node/console/power.go new file mode 100644 index 00000000..c105479a --- /dev/null +++ b/internal/app/wwctl/node/console/power.go @@ -0,0 +1,60 @@ +package console + +import ( + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/power" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var returnErr error = nil + var nodeList []node.NodeInfo + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) == 1 { + nodeList, _ = n.SearchByName(args[0]) + } else { + wwlog.Printf(wwlog.ERROR, "Please specify a node\n") + os.Exit(255) + } + + if len(nodeList) == 0 { + wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0]) + os.Exit(255) + } + + for _, node := range nodeList { + + if node.IpmiIpaddr.Get() == "" { + wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get()) + continue + } + + ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), + HostName: node.IpmiIpaddr.Get(), + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), + AuthType: "MD5", + } + + err := ipmiCmd.Console() + + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s: Console problem\n", node.Id.Get()) + returnErr = err + continue + } + + } + + return returnErr +} + diff --git a/internal/app/wwctl/node/console/root.go b/internal/app/wwctl/node/console/root.go new file mode 100644 index 00000000..f2515e5f --- /dev/null +++ b/internal/app/wwctl/node/console/root.go @@ -0,0 +1,19 @@ +package console + +import ( + "github.com/spf13/cobra" +) + +var ( + powerCmd = &cobra.Command{ + Use: "console", + Short: "IPMI console", + Long: "Start IPMI console for a singe node", + RunE: CobraRunE, + } +) + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return powerCmd +} diff --git a/internal/app/wwctl/node/powercycle/power.go b/internal/app/wwctl/node/powercycle/power.go new file mode 100644 index 00000000..54e6084b --- /dev/null +++ b/internal/app/wwctl/node/powercycle/power.go @@ -0,0 +1,81 @@ +package powercycle + +import ( + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/power" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/internal/pkg/batch" + "github.com/spf13/cobra" + "os" + "fmt" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var returnErr error = nil + var nodeList []node.NodeInfo + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) >= 1 { + nodeList, _ = n.SearchByNameList(args) + } else { + wwlog.Printf(wwlog.ERROR, "No requested nodes\n") + os.Exit(255) + } + + if len(nodeList) == 0 { + wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0]) + os.Exit(255) + } + + batchpool := batch.New(50, 0) + jobcount := len(nodeList) + results := make(chan power.IPMI, jobcount) + + for _, node := range nodeList { + + if node.IpmiIpaddr.Get() == "" { + wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get()) + continue + } + + ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), + HostName: node.IpmiIpaddr.Get(), + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), + AuthType: "MD5", + } + + batchpool.Submit(func() { + ipmiCmd.PowerCycle() + results <- ipmiCmd + }) + + } + + batchpool.Run() + + close(results) + + for result := range results { + + out, err := result.Result() + + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out) + returnErr = err + continue + } + + fmt.Printf("%s: %s\n", result.NodeName, out) + + } + + return returnErr +} + diff --git a/internal/app/wwctl/node/powercycle/root.go b/internal/app/wwctl/node/powercycle/root.go new file mode 100644 index 00000000..0adfee71 --- /dev/null +++ b/internal/app/wwctl/node/powercycle/root.go @@ -0,0 +1,19 @@ +package powercycle + +import ( + "github.com/spf13/cobra" +) + +var ( + powerCmd = &cobra.Command{ + Use: "powercycle", + Short: "power cycle node(s)", + Long: "cycle power for one or more nodes", + RunE: CobraRunE, + } +) + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return powerCmd +} diff --git a/internal/app/wwctl/node/poweroff/power.go b/internal/app/wwctl/node/poweroff/power.go index 42a13c91..62c6b2f5 100644 --- a/internal/app/wwctl/node/poweroff/power.go +++ b/internal/app/wwctl/node/poweroff/power.go @@ -4,8 +4,10 @@ import ( "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/power" "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/internal/pkg/batch" "github.com/spf13/cobra" "os" + "fmt" ) func CobraRunE(cmd *cobra.Command, args []string) error { @@ -28,10 +30,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error { if len(nodeList) == 0 { wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0]) os.Exit(255) - } else { - wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList)) } + batchpool := batch.New(50, 0) + jobcount := len(nodeList) + results := make(chan power.IPMI, jobcount) + for _, node := range nodeList { if node.IpmiIpaddr.Get() == "" { @@ -40,22 +44,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), HostName: node.IpmiIpaddr.Get(), - User: "ADMIN", - Password: "ADMIN", + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), AuthType: "MD5", } - out, err := ipmiCmd.PowerOff() + batchpool.Submit(func() { + ipmiCmd.PowerOff() + results <- ipmiCmd + }) + + } + + batchpool.Run() + + close(results) + + for result := range results { + + out, err := result.Result() if err != nil { - wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out) + wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out) returnErr = err continue } - wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out) + fmt.Printf("%s: %s\n", result.NodeName, out) + } return returnErr } + diff --git a/internal/app/wwctl/node/poweron/power.go b/internal/app/wwctl/node/poweron/power.go index 8232ecc4..da17a07a 100644 --- a/internal/app/wwctl/node/poweron/power.go +++ b/internal/app/wwctl/node/poweron/power.go @@ -4,8 +4,10 @@ import ( "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/power" "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/internal/pkg/batch" "github.com/spf13/cobra" "os" + "fmt" ) func CobraRunE(cmd *cobra.Command, args []string) error { @@ -32,9 +34,11 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList)) } - for _, node := range nodeList { + batchpool := batch.New(50, 0) + jobcount := len(nodeList) + results := make(chan power.IPMI, jobcount) - var powerCmd power.PowerOnInterface + for _, node := range nodeList { if node.IpmiIpaddr.Get() == "" { wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get()) @@ -42,24 +46,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), HostName: node.IpmiIpaddr.Get(), - User: "ADMIN", - Password: "ADMIN", + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), AuthType: "MD5", } - powerCmd = ipmiCmd + batchpool.Submit(func() { + ipmiCmd.PowerOn() + results <- ipmiCmd + }) - out, err := powerCmd.PowerOn() + } + + batchpool.Run() + + close(results) + + for result := range results { + + out, err := result.Result() if err != nil { - wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out) + wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out) returnErr = err continue } - wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out) + fmt.Printf("%s: %s\n", result.NodeName, out) + } return returnErr } + diff --git a/internal/app/wwctl/node/powerstatus/power.go b/internal/app/wwctl/node/powerstatus/power.go index 69b063ac..3449252e 100644 --- a/internal/app/wwctl/node/powerstatus/power.go +++ b/internal/app/wwctl/node/powerstatus/power.go @@ -4,8 +4,10 @@ import ( "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/power" "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/internal/pkg/batch" "github.com/spf13/cobra" "os" + "fmt" ) func CobraRunE(cmd *cobra.Command, args []string) error { @@ -32,6 +34,10 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Printf(wwlog.VERBOSE, "Found %d matching nodes for power command\n", len(nodeList)) } + batchpool := batch.New(50, 0) + jobcount := len(nodeList) + results := make(chan power.IPMI, jobcount) + for _, node := range nodeList { if node.IpmiIpaddr.Get() == "" { @@ -40,22 +46,38 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), HostName: node.IpmiIpaddr.Get(), - User: "ADMIN", - Password: "ADMIN", + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), AuthType: "MD5", } - out, err := ipmiCmd.PowerStatus() + batchpool.Submit(func() { + ipmiCmd.PowerStatus() + results <- ipmiCmd + }) + + } + + batchpool.Run() + + close(results) + + for result := range results { + + out, err := result.Result() if err != nil { - wwlog.Printf(wwlog.ERROR, "%s: %s\n", node.Id.Get(), out) + wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out) returnErr = err continue } - wwlog.Printf(wwlog.INFO, "%s: %s\n", node.Id.Get(), out) + fmt.Printf("%s: %s\n", result.NodeName, out) + } return returnErr } + diff --git a/internal/app/wwctl/node/root.go b/internal/app/wwctl/node/root.go index 8d7f4ab4..af8686fb 100644 --- a/internal/app/wwctl/node/root.go +++ b/internal/app/wwctl/node/root.go @@ -6,7 +6,10 @@ import ( "github.com/hpcng/warewulf/internal/app/wwctl/node/list" "github.com/hpcng/warewulf/internal/app/wwctl/node/poweron" "github.com/hpcng/warewulf/internal/app/wwctl/node/poweroff" + "github.com/hpcng/warewulf/internal/app/wwctl/node/powercycle" "github.com/hpcng/warewulf/internal/app/wwctl/node/powerstatus" + "github.com/hpcng/warewulf/internal/app/wwctl/node/sensors" + "github.com/hpcng/warewulf/internal/app/wwctl/node/console" "github.com/hpcng/warewulf/internal/app/wwctl/node/set" "github.com/spf13/cobra" ) @@ -22,11 +25,14 @@ var ( func init() { baseCmd.AddCommand(poweron.GetCommand()) baseCmd.AddCommand(poweroff.GetCommand()) + baseCmd.AddCommand(powercycle.GetCommand()) baseCmd.AddCommand(powerstatus.GetCommand()) + baseCmd.AddCommand(sensors.GetCommand()) baseCmd.AddCommand(list.GetCommand()) baseCmd.AddCommand(set.GetCommand()) baseCmd.AddCommand(add.GetCommand()) baseCmd.AddCommand(delete.GetCommand()) + baseCmd.AddCommand(console.GetCommand()) } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/node/sensors/power.go b/internal/app/wwctl/node/sensors/power.go new file mode 100644 index 00000000..2d8095b8 --- /dev/null +++ b/internal/app/wwctl/node/sensors/power.go @@ -0,0 +1,87 @@ +package sensors + +import ( + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/power" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/internal/pkg/batch" + "github.com/spf13/cobra" + "os" + "fmt" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var returnErr error = nil + var nodeList []node.NodeInfo + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) >= 1 { + nodeList, _ = n.SearchByNameList(args) + } else { + wwlog.Printf(wwlog.ERROR, "No requested nodes\n") + os.Exit(255) + } + + if len(nodeList) == 0 { + wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0]) + os.Exit(255) + } + + batchpool := batch.New(50, 0) + jobcount := len(nodeList) + results := make(chan power.IPMI, jobcount) + + for _, node := range nodeList { + + if node.IpmiIpaddr.Get() == "" { + wwlog.Printf(wwlog.ERROR, "%s: No IPMI IP address\n", node.Id.Get()) + continue + } + + ipmiCmd := power.IPMI{ + NodeName: node.Id.Get(), + HostName: node.IpmiIpaddr.Get(), + User: node.IpmiUserName.Get(), + Password: node.IpmiPassword.Get(), + AuthType: "MD5", + } + + fullFlag := full + + batchpool.Submit(func() { + if fullFlag == true { + ipmiCmd.SensorList() + } else { + ipmiCmd.SDRList() + } + results <- ipmiCmd + }) + + } + + batchpool.Run() + + close(results) + + for result := range results { + + out, err := result.Result() + + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s: %s\n", result.NodeName, out) + returnErr = err + continue + } + + fmt.Printf("%s:\n%s\n", result.NodeName, out) + + } + + return returnErr +} + diff --git a/internal/app/wwctl/node/sensors/root.go b/internal/app/wwctl/node/sensors/root.go new file mode 100644 index 00000000..a64a37e3 --- /dev/null +++ b/internal/app/wwctl/node/sensors/root.go @@ -0,0 +1,24 @@ +package sensors + +import ( + "github.com/spf13/cobra" +) + +var ( + powerCmd = &cobra.Command{ + Use: "sensors", + Short: "node sensor information", + Long: "Get Node Sensor Information", + RunE: CobraRunE, + } + full bool +) + +func init() { + powerCmd.PersistentFlags().BoolVarP(&full, "full", "F", false, "show detailed output.") +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return powerCmd +} diff --git a/internal/pkg/batch/batch.go b/internal/pkg/batch/batch.go new file mode 100644 index 00000000..c2179cea --- /dev/null +++ b/internal/pkg/batch/batch.go @@ -0,0 +1,57 @@ + +package batch + +import ( + "time" + "sync" +) + +type BatchPool struct { + active int + mintime int + jobcount int + jobs []func() +} + +func New(active int, mintime int) *BatchPool { + pool := &BatchPool{ + active: active, + mintime: mintime, + } + pool.jobs = []func(){} + return pool +} + +func Min(x, y int) int { + if x < y { + return x + } + return y +} + +func (pool *BatchPool) Submit(f func()) { + pool.jobcount++ + pool.jobs = append(pool.jobs, f) +} + +func wrapper(wg *sync.WaitGroup, f func()) { + defer wg.Done() + f() +} + +func (pool *BatchPool) Run() { + var wg sync.WaitGroup + count := pool.jobcount + for count > 0 { + for i:=0; i 0) { + time.Sleep(time.Second * time.Duration(pool.mintime)) + } + wg.Wait() + } +} + diff --git a/internal/pkg/power/ipmitool.go b/internal/pkg/power/ipmitool.go index e33c0f99..063fa6ea 100644 --- a/internal/pkg/power/ipmitool.go +++ b/internal/pkg/power/ipmitool.go @@ -2,16 +2,28 @@ package power import ( "os/exec" + "os" ) +type IPMIResult struct { + err error + out string +} + type IPMI struct { + NodeName string HostName string User string Password string AuthType string + result IPMIResult } -func (ipmi IPMI) Command(ipmiArgs []string) ([]byte, error) { +func (ipmi *IPMI) Result() (string, error) { + return ipmi.result.out, ipmi.result.err +} + +func (ipmi *IPMI) Command(ipmiArgs []string) ([]byte, error) { var args []string @@ -21,29 +33,67 @@ func (ipmi IPMI) Command(ipmiArgs []string) ([]byte, error) { return ipmiCmd.CombinedOutput() } -func (ipmi IPMI) PowerOn() (string, error) { +func (ipmi *IPMI) InteractiveCommand(ipmiArgs []string) error { var args []string - args = append(args, "chassis", "power", "on") - ipmiOut, err := ipmi.Command(args) - return string(ipmiOut), err + args = append(args, "-I", "lanplus", "-H", ipmi.HostName, "-U", ipmi.User, "-P", ipmi.Password) + args = append(args, ipmiArgs...) + ipmiCmd := exec.Command("/usr/bin/ipmitool", args...) + ipmiCmd.Stdout = os.Stdout + ipmiCmd.Stdin = os.Stdin + ipmiCmd.Stderr = os.Stderr + return ipmiCmd.Run() } -func (ipmi IPMI) PowerOff() (string, error) { - - var args []string - - args = append(args, "chassis", "power", "off") - ipmiOut, err := ipmi.Command(args) - return string(ipmiOut), err +func (ipmi *IPMI) IPMIInteractiveCommand(args ...string) error { + return ipmi.InteractiveCommand(args) } -func (ipmi IPMI) PowerStatus() (string, error) { - - var args []string - - args = append(args, "chassis", "power", "status") +func (ipmi *IPMI) IPMICommand(args ...string) (string, error) { ipmiOut, err := ipmi.Command(args) - return string(ipmiOut), err + ipmi.result.out = string(ipmiOut) + ipmi.result.err = err + return ipmi.result.out, ipmi.result.err + } + +//func (ipmi IPMI) PowerOn() (string, error) { +// +// var args []string +// +// args = append(args, "chassis", "power", "on") +// ipmiOut, err := ipmi.Command(args) +// ipmi.result.out = string(ipmiOut) +// ipmi.result.err = err +// return ipmi.result.out, ipmi.result.err +//} + +func (ipmi *IPMI) PowerOn() (string, error) { + return ipmi.IPMICommand("chassis", "power", "on") +} + +func (ipmi *IPMI) PowerOff() (string, error) { + return ipmi.IPMICommand("chassis", "power", "off") +} + +func (ipmi *IPMI) PowerCycle() (string, error) { + return ipmi.IPMICommand("chassis", "power", "cycle") +} + +func (ipmi *IPMI) PowerStatus() (string, error) { + return ipmi.IPMICommand("chassis", "power", "status") +} + +func (ipmi *IPMI) SDRList() (string, error) { + return ipmi.IPMICommand("sdr", "list") +} + +func (ipmi *IPMI) SensorList() (string, error) { + return ipmi.IPMICommand("sensor", "list") +} + +func (ipmi *IPMI) Console() error { + return ipmi.IPMIInteractiveCommand("sol", "activate") +} + diff --git a/overlays/system/default/ipmi.ww b/overlays/system/default/ipmi.ww index aa0605a9..5d10b075 100755 --- a/overlays/system/default/ipmi.ww +++ b/overlays/system/default/ipmi.ww @@ -84,11 +84,14 @@ fi #ipmitool user priv 1 4 1 # Serial Over LAN +ipmitool channel setaccess 1 2 link=on ipmi=on callin=on privilege=4 ipmitool sol set force-encryption true 1 ipmitool sol set force-authentication true 1 ipmitool sol set privilege-level admin 1 -ipmitool sol payload enable 1 4 -ipmitool sol set enabled true 1 -ipmitool sol set non-volatile-bit-rate 19.2 1 -ipmitool sol set volatile-bit-rate 19.2 1 +#ipmitool sol payload enable 1 4 +ipmitool sol payload enable 1 2 1 +ipmitool sol set enabled true 1 1 +speed=38.4 # 19.2 38.4 115.2 +ipmitool sol set non-volatile-bit-rate $speed 1 +ipmitool sol set volatile-bit-rate $speed 1 diff --git a/test/batch-test.go b/test/batch-test.go new file mode 100644 index 00000000..a0d93672 --- /dev/null +++ b/test/batch-test.go @@ -0,0 +1,51 @@ + +package main + +import ( + "github.com/hpcng/warewulf/internal/pkg/batch" + "fmt" + "time" + "runtime" +) + +func exampleJob(name string, id int) string { + fmt.Printf("[%d] %s started...\n", id, name) + time.Sleep(time.Second * 5) + fmt.Printf("[%d] %s ending...\n", id, name) + return name +} + + +func main() { + + fmt.Printf("GOMAXPROCS=%d\n", runtime.GOMAXPROCS(0)) + + jobstorun := 1000 + + // create batch pool to run 100 jobs at a time every 10 seconds + batchpool := batch.New(100, 10) + retvalues := make(chan string, jobstorun) + + // submit a bunch of jobs + for i:=0; i