From b4ef71216658d88264b899d0d0e84a1274a0bd9b Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Tue, 7 Jun 2022 20:28:25 -0700 Subject: [PATCH 1/2] Add parallel ssh capability --- internal/app/wwctl/root.go | 21 ++++---- internal/app/wwctl/ssh/main.go | 89 ++++++++++++++++++++++++++++++++++ internal/app/wwctl/ssh/root.go | 31 ++++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 internal/app/wwctl/ssh/main.go create mode 100644 internal/app/wwctl/ssh/root.go diff --git a/internal/app/wwctl/root.go b/internal/app/wwctl/root.go index e724e43b..81fe2a26 100644 --- a/internal/app/wwctl/root.go +++ b/internal/app/wwctl/root.go @@ -9,9 +9,10 @@ import ( "github.com/hpcng/warewulf/internal/app/wwctl/power" "github.com/hpcng/warewulf/internal/app/wwctl/profile" "github.com/hpcng/warewulf/internal/app/wwctl/server" + "github.com/hpcng/warewulf/internal/app/wwctl/ssh" "github.com/hpcng/warewulf/internal/app/wwctl/version" - "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/hpcng/warewulf/internal/pkg/help" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" @@ -21,12 +22,12 @@ import ( var ( rootCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "wwctl COMMAND [OPTIONS]", - Short: "Warewulf Control", - Long: "Control interface to the Warewulf Cluster Provisioning System.", - PersistentPreRunE: rootPersistentPreRunE, - SilenceUsage: true, - SilenceErrors: true, + Use: "wwctl COMMAND [OPTIONS]", + Short: "Warewulf Control", + Long: "Control interface to the Warewulf Cluster Provisioning System.", + PersistentPreRunE: rootPersistentPreRunE, + SilenceUsage: true, + SilenceErrors: true, } verboseArg bool DebugFlag bool @@ -36,8 +37,8 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.") rootCmd.PersistentFlags().BoolVarP(&DebugFlag, "debug", "d", false, "Run with debugging messages enabled.") - rootCmd.SetUsageTemplate(help.UsageTemplate) - rootCmd.SetHelpTemplate(help.HelpTemplate) + rootCmd.SetUsageTemplate(help.UsageTemplate) + rootCmd.SetHelpTemplate(help.HelpTemplate) rootCmd.AddCommand(overlay.GetCommand()) rootCmd.AddCommand(container.GetCommand()) @@ -48,7 +49,7 @@ func init() { rootCmd.AddCommand(configure.GetCommand()) rootCmd.AddCommand(server.GetCommand()) rootCmd.AddCommand(version.GetCommand()) - + rootCmd.AddCommand(ssh.GetCommand()) } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/ssh/main.go b/internal/app/wwctl/ssh/main.go new file mode 100644 index 00000000..0e6a194f --- /dev/null +++ b/internal/app/wwctl/ssh/main.go @@ -0,0 +1,89 @@ +package ssh + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "time" + + "github.com/hpcng/warewulf/internal/pkg/batch" + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/hpcng/warewulf/pkg/hostlist" + "github.com/spf13/cobra" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + batchpool := batch.New(FanOut) + + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + nodes, err := nodeDB.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) + os.Exit(1) + } + + if len(args) > 0 { + nodes = node.FilterByName(nodes, hostlist.Expand(args)) + } else { + //nolint:errcheck + cmd.Usage() + os.Exit(1) + } + + for _, node := range nodes { + + if _, ok := node.NetDevs["default"]; !ok { + fmt.Fprintf(os.Stderr, "%s: Default network device doesn't exist\n", node.Id.Get()) + continue + } + + if node.NetDevs["default"].Ipaddr.Get() == "" { + fmt.Fprintf(os.Stderr, "%s: Default network IP address not configured\n", node.Id.Get()) + continue + } + + nodename := node.Id.Print() + var command []string + + command = append(command, node.NetDevs["default"].Ipaddr.Get()) + command = append(command, args[1:]...) + + batchpool.Submit(func() { + + wwlog.Printf(wwlog.DEBUG, "Sending command to node '%s': %s\n", nodename, command) + var stdout, stderr bytes.Buffer + cmd := exec.Command(SshPath, command...) + cmd.Stdin = os.Stdin + cmd.Stdout = &stdout + cmd.Stderr = &stderr + _ = cmd.Run() + + scan_stdout := bufio.NewScanner(&stdout) + for scan_stdout.Scan() { + fmt.Printf("%s: %s\n", nodename, scan_stdout.Text()) + } + + scan_stderr := bufio.NewScanner(&stderr) + for scan_stderr.Scan() { + fmt.Fprintf(os.Stderr, "%s: %s\n", nodename, scan_stderr.Text()) + } + + //util.ExecInteractive(SshPath, command...) + time.Sleep(time.Duration(Sleep) * time.Second) + + }) + + } + + batchpool.Run() + + return nil +} diff --git a/internal/app/wwctl/ssh/root.go b/internal/app/wwctl/ssh/root.go new file mode 100644 index 00000000..2c716b86 --- /dev/null +++ b/internal/app/wwctl/ssh/root.go @@ -0,0 +1,31 @@ +package ssh + +import ( + "github.com/spf13/cobra" +) + +var ( + baseCmd = &cobra.Command{ + DisableFlagsInUseLine: true, + Use: "ssh [OPTIONS] NODE_PATTERN...", + Short: "SSH into configured nodes in parallel", + Long: "Easily ssh into nodes in parallel to run non-interactive commands\n", + RunE: CobraRunE, + } + Show bool + FanOut int + Sleep int + SshPath string +) + +func init() { + baseCmd.PersistentFlags().BoolVarP(&Show, "dryrun", "n", false, "Show commands to run") + baseCmd.PersistentFlags().IntVarP(&FanOut, "fanout", "f", 32, "How many connections to run in parallel") + baseCmd.PersistentFlags().IntVarP(&Sleep, "sleep", "s", 0, "Seconds to sleep inbetween processes") + baseCmd.PersistentFlags().StringVar(&SshPath, "rsh", "/usr/bin/ssh", "Path to use for RSH/SSH command") +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} From 3eabf47fc6b3d5d180af44385f629196c6357f09 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Tue, 7 Jun 2022 20:34:26 -0700 Subject: [PATCH 2/2] Fix dryrun/show for ssh command --- internal/app/wwctl/ssh/main.go | 37 +++++++++++++++++++--------------- internal/app/wwctl/ssh/root.go | 4 ++-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/internal/app/wwctl/ssh/main.go b/internal/app/wwctl/ssh/main.go index 0e6a194f..c43e6142 100644 --- a/internal/app/wwctl/ssh/main.go +++ b/internal/app/wwctl/ssh/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "strings" "time" "github.com/hpcng/warewulf/internal/pkg/batch" @@ -58,24 +59,28 @@ func CobraRunE(cmd *cobra.Command, args []string) error { batchpool.Submit(func() { - wwlog.Printf(wwlog.DEBUG, "Sending command to node '%s': %s\n", nodename, command) - var stdout, stderr bytes.Buffer - cmd := exec.Command(SshPath, command...) - cmd.Stdin = os.Stdin - cmd.Stdout = &stdout - cmd.Stderr = &stderr - _ = cmd.Run() + if DryRun { + fmt.Printf("%s: %s %s\n", nodename, SshPath, strings.Join(command, " ")) + } else { - scan_stdout := bufio.NewScanner(&stdout) - for scan_stdout.Scan() { - fmt.Printf("%s: %s\n", nodename, scan_stdout.Text()) + wwlog.Printf(wwlog.DEBUG, "Sending command to node '%s': %s\n", nodename, command) + var stdout, stderr bytes.Buffer + cmd := exec.Command(SshPath, command...) + cmd.Stdin = os.Stdin + cmd.Stdout = &stdout + cmd.Stderr = &stderr + _ = cmd.Run() + + scan_stdout := bufio.NewScanner(&stdout) + for scan_stdout.Scan() { + fmt.Printf("%s: %s\n", nodename, scan_stdout.Text()) + } + + scan_stderr := bufio.NewScanner(&stderr) + for scan_stderr.Scan() { + fmt.Fprintf(os.Stderr, "%s: %s\n", nodename, scan_stderr.Text()) + } } - - scan_stderr := bufio.NewScanner(&stderr) - for scan_stderr.Scan() { - fmt.Fprintf(os.Stderr, "%s: %s\n", nodename, scan_stderr.Text()) - } - //util.ExecInteractive(SshPath, command...) time.Sleep(time.Duration(Sleep) * time.Second) diff --git a/internal/app/wwctl/ssh/root.go b/internal/app/wwctl/ssh/root.go index 2c716b86..e5eaec93 100644 --- a/internal/app/wwctl/ssh/root.go +++ b/internal/app/wwctl/ssh/root.go @@ -12,14 +12,14 @@ var ( Long: "Easily ssh into nodes in parallel to run non-interactive commands\n", RunE: CobraRunE, } - Show bool + DryRun bool FanOut int Sleep int SshPath string ) func init() { - baseCmd.PersistentFlags().BoolVarP(&Show, "dryrun", "n", false, "Show commands to run") + baseCmd.PersistentFlags().BoolVarP(&DryRun, "dryrun", "n", false, "Show commands to run") baseCmd.PersistentFlags().IntVarP(&FanOut, "fanout", "f", 32, "How many connections to run in parallel") baseCmd.PersistentFlags().IntVarP(&Sleep, "sleep", "s", 0, "Seconds to sleep inbetween processes") baseCmd.PersistentFlags().StringVar(&SshPath, "rsh", "/usr/bin/ssh", "Path to use for RSH/SSH command")