Lots of changes including: wwctl overlay list and framework for other commands

cmd/wwclient/wwclient.go

@@ -19,7 +19,7 @@ func main() {
 		time.Sleep(5000 * time.Millisecond)
 	} else {
 		fmt.Printf("Called via: %s\n", os.Args[0])
-		fmt.Printf("Runtime system-overlay is being put in '/warewulf/wwclient-test' rather than '/'\n")
+		fmt.Printf("Runtime overlay is being put in '/warewulf/wwclient-test' rather than '/'\n")
 		os.MkdirAll("/warewulf/wwclient-test", 0755)
 		os.Chdir("/warewulf/wwclient-test")
 	}
@@ -74,12 +74,12 @@ func main() {
 		}
 
 		if resp.StatusCode != 200 {
-			log.Printf("Not updating runtime system-overlay, got status code: %d\n", resp.StatusCode)
+			log.Printf("Not updating runtime overlay, got status code: %d\n", resp.StatusCode)
 			time.Sleep(60000 * time.Millisecond)
 			continue
 		}
 
-		log.Printf("Updating runtime system\n")
+		log.Printf("Updating system\n")
 		command := exec.Command("/bin/cpio", "-iu")
 		command.Stdin = resp.Body
 		err := command.Run()

internal/app/warewulfd/root.go

@@ -6,7 +6,7 @@ import (
 )
 
 var (
-	WarewulfdCmd = &cobra.Command{
+	baseCmd = &cobra.Command{
 		Use:                "warewulfd",
 		Short:              "Warewulf Daemon Service",
 		Long:               "This is the primary Warewulf service for provisioning nodes",
@@ -19,13 +19,13 @@ var (
 
 
 func init() {
-	WarewulfdCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.")
-	WarewulfdCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.")
+	baseCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.")
+	baseCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.")
 }
 
 // GetRootCommand returns the root cobra.Command for the application.
 func GetRootCommand() *cobra.Command {
-	return WarewulfdCmd
+	return baseCmd
 }
 
 func rootPersistentPreRunE(cmd *cobra.Command, args []string) error {

internal/app/wwctl/build/root.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	buildCmd = &cobra.Command{
+	baseCmd = &cobra.Command{
 		Use:                "build",
 		Short:              "Warewulf build subcommand",
 		Long:               "Warewulf build is used to build VNFS, kernel, and system-overlay objects for\n" +
@@ -22,16 +22,16 @@ var (
 )
 
 func init() {
-	buildCmd.PersistentFlags().BoolVarP(&buildVnfs, "vnfs", "V", false, "Build and/or update VNFS images.")
-	buildCmd.PersistentFlags().BoolVarP(&buildKernel, "kernel", "K", false, "Build and/or update Kernel images.")
-	buildCmd.PersistentFlags().BoolVarP(&buildRuntimeOverlay, "runtime", "R", false, "Build and/or update runtime overlays")
-	buildCmd.PersistentFlags().BoolVarP(&buildSystemOverlay, "system", "S", false, "Build and/or update system overlays")
-	buildCmd.PersistentFlags().BoolVarP(&buildAll, "all", "A", false, "Build and/or update all components")
-	buildCmd.PersistentFlags().BoolVarP(&buildForce, "force", "f", false, "Force build even if nothing has been updated.")
+	baseCmd.PersistentFlags().BoolVarP(&buildVnfs, "vnfs", "V", false, "Build and/or update VNFS images.")
+	baseCmd.PersistentFlags().BoolVarP(&buildKernel, "kernel", "K", false, "Build and/or update Kernel images.")
+	baseCmd.PersistentFlags().BoolVarP(&buildRuntimeOverlay, "runtime", "R", false, "Build and/or update runtime overlays")
+	baseCmd.PersistentFlags().BoolVarP(&buildSystemOverlay, "system", "S", false, "Build and/or update system overlays")
+	baseCmd.PersistentFlags().BoolVarP(&buildAll, "all", "A", false, "Build and/or update all components")
+	baseCmd.PersistentFlags().BoolVarP(&buildForce, "force", "f", false, "Force build even if nothing has been updated.")
 
 }
 
 // GetRootCommand returns the root cobra.Command for the application.
 func GetCommand() *cobra.Command {
-	return buildCmd
+	return baseCmd
 }

internal/app/wwctl/kernel/root.go

@@ -0,0 +1,31 @@
+package kernel
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "kernel",
+		Short:              "Kernel Image Management",
+		Long:               "Management of Warewulf Kernels to be used for bootstrapping nodes",
+		RunE:				CobraRunE,
+	}
+	test bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	fmt.Printf("Kernel: Hello World\n")
+	return nil
+}

internal/app/wwctl/node/power/root.go

@@ -0,0 +1,31 @@
+package power
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "power",
+		Short:              "Node power management",
+		Long:               "Node Power management commands",
+		RunE:				CobraRunE,
+	}
+	test bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	fmt.Printf("Power: Hello World\n")
+	return nil
+}

internal/app/wwctl/node/root.go

@@ -0,0 +1,27 @@
+package node
+
+import (
+	"github.com/hpcng/warewulf/internal/app/wwctl/node/power"
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "node",
+		Short:              "Node management",
+		Long:               "Management of node settings and power management",
+	}
+	test bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+
+	baseCmd.AddCommand(power.GetCommand())
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}

internal/app/wwctl/overlay/create/main.go

@@ -0,0 +1,31 @@
+package create
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/overlay"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/spf13/cobra"
+	"os"
+)
+
+
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+
+	if SystemOverlay == true {
+		err := overlay.SystemOverlayInit(args[0])
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "%s\n", err)
+			os.Exit(1)
+		}
+		wwlog.Printf(wwlog.INFO, "Created new system overlay: %s\n", args[0])
+	} else {
+		err := overlay.RuntimeOverlayInit(args[0])
+		if err != nil {
+			wwlog.Printf(wwlog.ERROR, "%s\n", err)
+			os.Exit(1)
+		}
+		wwlog.Printf(wwlog.INFO, "Created new runtime overlay: %s\n", args[0])
+	}
+
+	return nil
+}

internal/app/wwctl/overlay/create/root.go

@@ -0,0 +1,25 @@
+package create
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "create",
+		Short:              "Initialize a new Overlay",
+		Long:               "Create a new Warewulf provisioning overlay",
+		RunE:				CobraRunE,
+	}
+	SystemOverlay bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&SystemOverlay, "system", "s", false, "Show System Overlays as well")
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}

internal/app/wwctl/overlay/edit/root.go

@@ -0,0 +1,35 @@
+package edit
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "edit",
+		Short:              "Edit Warewulf Overlay files",
+		Long:               "Warewulf edit overlay files",
+		RunE:				CobraRunE,
+	}
+	SystemOverlay bool
+	ListFiles bool
+
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&SystemOverlay, "system", "s", false, "Show system overlays instead of runtime")
+	baseCmd.PersistentFlags().BoolVarP(&ListFiles, "files", "f", false, "List files contained within a given overlay")
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}
+
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	fmt.Printf("Edit: Hello World\n")
+	return nil
+}

internal/app/wwctl/overlay/list/main.go

@@ -0,0 +1,89 @@
+package list
+
+import (
+	"fmt"
+	"github.com/hpcng/warewulf/internal/pkg/assets"
+	"github.com/hpcng/warewulf/internal/pkg/config"
+	"github.com/hpcng/warewulf/internal/pkg/overlay"
+	"github.com/hpcng/warewulf/internal/pkg/util"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/spf13/cobra"
+)
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	config := config.New()
+	set := make(map[string]int)
+	var o []string
+	var err error
+	var nodeList []assets.NodeInfo
+
+	if SystemOverlay == true {
+		fmt.Printf("%-25s %-8s %-8s\n", "SYSTEM OVERLAY NAME", "NODES", "FILES")
+		o, err = overlay.FindAllSystemOverlays()
+	} else {
+		fmt.Printf("%-25s %-8s %-8s\n", "RUNTIME OVERLAY NAME", "NODES", "FILES")
+		o, err = overlay.FindAllRuntimeOverlays()
+	}
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not get system overlays: %s\n", err)
+		return err
+	}
+
+	nodeList, err = assets.FindAllNodes()
+	if err != nil {
+		wwlog.Printf(wwlog.ERROR, "Could not get node configuration: %s\n", err)
+		return err
+	}
+
+	for _, node := range nodeList {
+		if SystemOverlay == true {
+			if node.SystemOverlay != "" {
+				set[node.SystemOverlay] ++
+			}
+		} else {
+			if node.RuntimeOverlay != "" {
+				set[node.RuntimeOverlay] ++
+			}
+		}
+	}
+
+	for overlay := range o {
+		var path string
+		name := o[overlay]
+
+		if len(args) > 0 {
+			if args[0] != name {
+				continue
+			}
+		}
+
+		if SystemOverlay == true {
+			path = config.SystemOverlaySource(o[overlay])
+		} else {
+			path = config.RuntimeOverlaySource(o[overlay])
+		}
+
+		if util.IsDir(path) == true {
+			files := util.FindFiles(path)
+
+			wwlog.Printf(wwlog.DEBUG, "Iterating overlay path: %s\n", path)
+			if ListFiles == true {
+				var fileCount int
+				for file := range files {
+					fmt.Printf("%-25s %-8d /%s\n", name, set[name], files[file])
+					fileCount ++
+				}
+				if fileCount == 0 {
+					fmt.Printf("%-25s %-8d %-8d\n", name, set[name], 0)
+				}
+			} else {
+				fmt.Printf("%-25s %-8d %-8d\n", name, set[name], len(files))
+			}
+
+		} else {
+			wwlog.Printf(wwlog.ERROR, "system/%s (path not found:%s)\n", o[overlay], path)
+		}
+	}
+
+	return nil
+}

internal/app/wwctl/overlay/list/root.go

@@ -1,32 +1,27 @@
 package list
 
 import (
-	"fmt"
 	"github.com/spf13/cobra"
 )
 
 var (
-	listCmd = &cobra.Command{
+	baseCmd = &cobra.Command{
 		Use:                "list",
 		Short:              "List Warewulf Overlays",
 		Long:               "Warewulf List overlay",
 		RunE:				CobraRunE,
 	}
-
+	SystemOverlay bool
+	ListFiles bool
 )
 
 func init() {
+	baseCmd.PersistentFlags().BoolVarP(&SystemOverlay, "system", "s", false, "Show system overlays instead of runtime")
+	baseCmd.PersistentFlags().BoolVarP(&ListFiles, "files", "f", false, "List files contained within a given overlay")
 
 }
 
 // GetRootCommand returns the root cobra.Command for the application.
 func GetCommand() *cobra.Command {
-	return listCmd
-}
-
-
-func CobraRunE(cmd *cobra.Command, args []string) error {
-
-	fmt.Printf("List: Hello World\n")
-	return nil
+	return baseCmd
 }

internal/app/wwctl/overlay/root.go

@@ -1,6 +1,8 @@
 package overlay
 
 import (
+	"github.com/hpcng/warewulf/internal/app/wwctl/overlay/create"
+	"github.com/hpcng/warewulf/internal/app/wwctl/overlay/edit"
 	"github.com/hpcng/warewulf/internal/app/wwctl/overlay/list"
 	"github.com/hpcng/warewulf/internal/app/wwctl/overlay/show"
 
@@ -8,7 +10,7 @@ import (
 )
 
 var (
-	overlayCmd = &cobra.Command{
+	baseCmd = &cobra.Command{
 		Use:                "overlay",
 		Short:              "Warewulf Overlay Management",
 		Long:               "Management interface for Warewulf overlays",
@@ -17,14 +19,17 @@ var (
 )
 
 func init() {
-	overlayCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+//	baseCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+
+	baseCmd.AddCommand(list.GetCommand())
+	baseCmd.AddCommand(show.GetCommand())
+	baseCmd.AddCommand(create.GetCommand())
+	baseCmd.AddCommand(edit.GetCommand())
 
-	overlayCmd.AddCommand(list.GetCommand())
-	overlayCmd.AddCommand(show.GetCommand())
 
 }
 
 // GetRootCommand returns the root cobra.Command for the application.
 func GetCommand() *cobra.Command {
-	return overlayCmd
+	return baseCmd
 }

internal/app/wwctl/overlay/show/root.go

@@ -6,7 +6,7 @@ import (
 )
 
 var (
-	showCmd = &cobra.Command{
+	baseCmd = &cobra.Command{
 		Use:                "show",
 		Short:              "Show Warewulf Overlay objects",
 		Long:               "Warewulf show overlay objects",
@@ -21,12 +21,11 @@ func init() {
 
 // GetRootCommand returns the root cobra.Command for the application.
 func GetCommand() *cobra.Command {
-	return showCmd
+	return baseCmd
 }
 
 
 func CobraRunE(cmd *cobra.Command, args []string) error {
-
 	fmt.Printf("Show: Hello World\n")
 	return nil
 }

internal/app/wwctl/root.go

@@ -2,7 +2,10 @@ package wwctl
 
 import (
 	"github.com/hpcng/warewulf/internal/app/wwctl/build"
+	"github.com/hpcng/warewulf/internal/app/wwctl/kernel"
+	"github.com/hpcng/warewulf/internal/app/wwctl/node"
 	"github.com/hpcng/warewulf/internal/app/wwctl/overlay"
+	"github.com/hpcng/warewulf/internal/app/wwctl/vnfs"
 	"github.com/hpcng/warewulf/internal/pkg/wwlog"
 	"github.com/spf13/cobra"
 )
@@ -24,6 +27,9 @@ func init() {
 
 	rootCmd.AddCommand(build.GetCommand())
 	rootCmd.AddCommand(overlay.GetCommand())
+	rootCmd.AddCommand(vnfs.GetCommand())
+	rootCmd.AddCommand(node.GetCommand())
+	rootCmd.AddCommand(kernel.GetCommand())
 }
 
 // GetRootCommand returns the root cobra.Command for the application.

internal/app/wwctl/vnfs/root.go

@@ -0,0 +1,31 @@
+package vnfs
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+var (
+	baseCmd = &cobra.Command{
+		Use:                "vnfs",
+		Short:              "VNFS image management",
+		Long:               "Virtual Node File System (VNFS) image management",
+		RunE:				CobraRunE,
+	}
+	test bool
+)
+
+func init() {
+	baseCmd.PersistentFlags().BoolVarP(&test, "test", "t", false, "Testing.")
+
+}
+
+// GetRootCommand returns the root cobra.Command for the application.
+func GetCommand() *cobra.Command {
+	return baseCmd
+}
+
+func CobraRunE(cmd *cobra.Command, args []string) error {
+	fmt.Printf("Vnfs: Hello World\n")
+	return nil
+}

internal/pkg/overlay/overlay.go

@@ -0,0 +1,3 @@
+package overlay
+
+//Shared code will go here....

internal/pkg/overlay/runtime.go

@@ -0,0 +1,45 @@
+package overlay
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/config"
+	"github.com/hpcng/warewulf/internal/pkg/errors"
+	"github.com/hpcng/warewulf/internal/pkg/util"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"io/ioutil"
+	"os"
+)
+
+func FindAllRuntimeOverlays() ([]string, error) {
+	config := config.New()
+	var ret []string
+
+	wwlog.Printf(wwlog.DEBUG, "Looking for runtime overlays...")
+	files, err := ioutil.ReadDir(config.RuntimeOverlayDir())
+	if err != nil {
+		return ret, err
+	}
+
+	for _, file := range files {
+		wwlog.Printf(wwlog.DEBUG, "Evaluating runtime overlay: %s\n", file.Name())
+		if file.IsDir() == true {
+			ret = append(ret, file.Name())
+		}
+	}
+
+	return ret, nil
+}
+
+
+func RuntimeOverlayInit(name string) error {
+	config := config.New()
+
+	path := config.RuntimeOverlaySource(name)
+
+	if util.IsDir(path) == true {
+		return errors.New("Runtime overlay already exists: "+name)
+	}
+
+	err := os.MkdirAll(path, 0755)
+
+	return err
+}

internal/pkg/overlay/system.go

@@ -0,0 +1,45 @@
+package overlay
+
+import (
+	"github.com/hpcng/warewulf/internal/pkg/config"
+	"github.com/hpcng/warewulf/internal/pkg/errors"
+	"github.com/hpcng/warewulf/internal/pkg/util"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"io/ioutil"
+	"os"
+)
+
+func FindAllSystemOverlays() ([]string, error) {
+	config := config.New()
+	var ret []string
+
+	wwlog.Printf(wwlog.DEBUG, "Looking for system overlays...")
+	files, err := ioutil.ReadDir(config.SystemOverlayDir())
+	if err != nil {
+		return ret, err
+	}
+
+	for _, file := range files {
+		wwlog.Printf(wwlog.DEBUG, "Evaluating system overlay: %s\n", file.Name())
+		if file.IsDir() == true {
+			ret = append(ret, file.Name())
+		}
+	}
+
+	return ret, nil
+}
+
+
+func SystemOverlayInit(name string) error {
+	config := config.New()
+
+	path := config.SystemOverlaySource(name)
+
+	if util.IsDir(path) == true {
+		return errors.New("Runtime overlay already exists: "+name)
+	}
+
+	err := os.MkdirAll(path, 0755)
+
+	return err
+}

internal/pkg/util/util.go

@@ -6,6 +6,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	//   "strings"
@@ -81,3 +82,54 @@ func CopyFile(source string, dest string) error {
 
 	return destFD.Close()
 }
+
+
+func IsDir(path string) (bool) {
+	if stat, err := os.Stat(path); err == nil && stat.IsDir() {
+		return true
+	}
+	return false
+}
+
+
+func TaintCheck(pattern string, expr string) bool {
+	if b, _ := regexp.MatchString(expr, pattern); b == true {
+		return true
+	}
+	return false
+}
+
+func ValidateOrDie(hostname string, name string, pattern string, expr string) {
+	if TaintCheck(pattern, expr) == false {
+		wwlog.Printf(wwlog.ERROR, "Entry '%s:%s' contains illegal characters: '%s'\n", hostname, name, pattern)
+		os.Exit(1)
+	}
+}
+
+func FindFiles(path string) []string {
+	var ret []string
+
+	wwlog.Printf(wwlog.DEBUG, "Changing directory to FindFiles path: %s\n", path)
+	err := os.Chdir(path)
+	if err != nil {
+		wwlog.Printf(wwlog.WARN, "Could not chdir() to: %s\n", path)
+		return ret
+	}
+
+	err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if IsDir(location) == false {
+			wwlog.Printf(wwlog.DEBUG, "FindFiles() found: %s\n", location)
+			ret = append(ret, location)
+		}
+		return nil
+	})
+	if err != nil {
+		return ret
+	}
+
+	return ret
+}