From 0095b5562402591407cbcbdb374e7fda62bf4ccb Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sat, 5 Dec 2020 16:43:30 -0800 Subject: [PATCH 1/9] Updates around VNFS restructure and kernel restructure. More coming. --- internal/app/warewulfd/response/vnfs.go | 13 +- internal/app/wwctl/kernel/build/main.go | 99 +++++---- internal/app/wwctl/kernel/build/root.go | 17 +- internal/app/wwctl/kernel/list/main.go | 75 ++++--- internal/app/wwctl/node/set/main.go | 18 ++ internal/app/wwctl/node/set/root.go | 2 + internal/app/wwctl/profile/set/main.go | 19 ++ internal/app/wwctl/profile/set/root.go | 2 + internal/app/wwctl/ready/main.go | 12 +- internal/app/wwctl/vnfs/build/main.go | 107 +++++----- internal/app/wwctl/vnfs/list/main.go | 100 +++++---- internal/app/wwctl/vnfs/list/root.go | 10 +- internal/app/wwctl/vnfs/pull/main.go | 54 +++++ internal/app/wwctl/vnfs/pull/root.go | 26 +++ internal/app/wwctl/vnfs/root.go | 10 +- internal/pkg/config/config.go | 84 +------- internal/pkg/kernel/kernel.go | 64 +++++- internal/pkg/oci/cache.go | 10 +- internal/pkg/oci/puller.go | 16 +- internal/pkg/overlay/funcmap.go | 13 +- internal/pkg/util/util.go | 4 +- internal/pkg/vnfs/build.go | 54 +++++ internal/pkg/vnfs/pull.go | 45 +++++ internal/pkg/vnfs/vnfs.go | 189 +++++------------- .../pkg/{vnfs => vnfs_old}/containerdir.go | 0 internal/pkg/{vnfs => vnfs_old}/docker.go | 0 internal/pkg/{vnfs => vnfs_old}/util.go | 0 internal/pkg/vnfs_old/vnfs.go | 173 ++++++++++++++++ 28 files changed, 791 insertions(+), 425 deletions(-) create mode 100644 internal/app/wwctl/vnfs/pull/main.go create mode 100644 internal/app/wwctl/vnfs/pull/root.go create mode 100644 internal/pkg/vnfs/build.go create mode 100644 internal/pkg/vnfs/pull.go rename internal/pkg/{vnfs => vnfs_old}/containerdir.go (100%) rename internal/pkg/{vnfs => vnfs_old}/docker.go (100%) rename internal/pkg/{vnfs => vnfs_old}/util.go (100%) create mode 100644 internal/pkg/vnfs_old/vnfs.go diff --git a/internal/app/warewulfd/response/vnfs.go b/internal/app/warewulfd/response/vnfs.go index 5ef72c0e..38ac1c60 100644 --- a/internal/app/warewulfd/response/vnfs.go +++ b/internal/app/warewulfd/response/vnfs.go @@ -16,19 +16,14 @@ func VnfsSend(w http.ResponseWriter, req *http.Request) { } if node.Vnfs.Defined() == true { + vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) - v, err := vnfs.Load(node.Vnfs.Get()) - if err != nil { - w.WriteHeader(503) - log.Printf("ERROR: Could not load VNFS: %s\n", node.Id.Get()) - return - } - - err = sendFile(w, v.Image, node.Id.Get()) + err = sendFile(w, vnfsImage, node.Id.Get()) if err != nil { log.Printf("ERROR1: %s\n", err) + w.WriteHeader(503) } else { - log.Printf("SEND: %15s: %s\n", node.Id.Get(), v.Image) + log.Printf("SEND: %15s: %s\n", node.Id.Get(), vnfsImage) } } else { w.WriteHeader(503) diff --git a/internal/app/wwctl/kernel/build/main.go b/internal/app/wwctl/kernel/build/main.go index 25d0ef01..1e1c474d 100644 --- a/internal/app/wwctl/kernel/build/main.go +++ b/internal/app/wwctl/kernel/build/main.go @@ -2,66 +2,77 @@ package build import ( "github.com/hpcng/warewulf/internal/pkg/kernel" - "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" ) func CobraRunE(cmd *cobra.Command, args []string) error { - var nodes []node.NodeInfo - set := make(map[string]int) - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) + for _, arg := range args { + err := kernel.Build(arg) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Failed building kernel: %s\n", err) + os.Exit(1) + } } - if len(args) == 1 && ByNode == true { - var err error - nodes, err = n.SearchByName(args[0]) + /* + var nodes []node.NodeInfo + set := make(map[string]int) + + n, err := node.New() if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) os.Exit(1) } - for _, node := range nodes { - if node.KernelVersion.Defined() == true { - set[node.KernelVersion.Get()]++ + if len(args) == 1 && ByNode == true { + var err error + nodes, err = n.SearchByName(args[0]) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + os.Exit(1) + } + + for _, node := range nodes { + if node.KernelVersion.Defined() == true { + set[node.KernelVersion.Get()]++ + } + } + + } else if BuildAll == true { + var err error + nodes, err = n.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) + os.Exit(1) + } + + for _, node := range nodes { + wwlog.Printf(wwlog.DEBUG, "evaluating node/kernel: %s/%s\n", node.Id.Get(), node.KernelVersion.Get()) + if node.KernelVersion.Defined() == true { + set[node.KernelVersion.Get()]++ + } + } + + } else if len(args) == 1 { + set[args[0]]++ + } else { + cmd.Usage() + os.Exit(1) + } + + for k := range set { + wwlog.Printf(wwlog.INFO, "Building kernel: %s\n", k) + err := kernel.Build(k) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) } } - } else if BuildAll == true { - var err error - nodes, err = n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) - os.Exit(1) - } - - for _, node := range nodes { - wwlog.Printf(wwlog.DEBUG, "evaluating node/kernel: %s/%s\n", node.Id.Get(), node.KernelVersion.Get()) - if node.KernelVersion.Defined() == true { - set[node.KernelVersion.Get()]++ - } - } - - } else if len(args) == 1 { - set[args[0]]++ - } else { - cmd.Usage() - os.Exit(1) - } - - for k := range set { - wwlog.Printf(wwlog.INFO, "Building kernel: %s\n", k) - err := kernel.Build(k) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } + */ return nil } diff --git a/internal/app/wwctl/kernel/build/root.go b/internal/app/wwctl/kernel/build/root.go index 5bce78c6..82f04fdf 100644 --- a/internal/app/wwctl/kernel/build/root.go +++ b/internal/app/wwctl/kernel/build/root.go @@ -1,20 +1,19 @@ package build import ( -"github.com/spf13/cobra" + "github.com/spf13/cobra" ) var ( baseCmd = &cobra.Command{ - Use: "build (kernel version | node search pattern)", - Short: "Kernel Image Build", - Long: "Build kernel images", - RunE: CobraRunE, - Args: cobra.RangeArgs(0,1), - + Use: "build (kernel version | node search pattern)", + Short: "Kernel Image Build", + Long: "Build kernel images", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), } BuildAll bool - ByNode bool + ByNode bool ) func init() { @@ -25,4 +24,4 @@ func init() { // GetRootCommand returns the root cobra.Command for the application. func GetCommand() *cobra.Command { return baseCmd -} \ No newline at end of file +} diff --git a/internal/app/wwctl/kernel/list/main.go b/internal/app/wwctl/kernel/list/main.go index 87b25e76..4b2132c7 100644 --- a/internal/app/wwctl/kernel/list/main.go +++ b/internal/app/wwctl/kernel/list/main.go @@ -2,17 +2,22 @@ package list import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" - "io/ioutil" "os" - "path" - "strings" ) func CobraRunE(cmd *cobra.Command, args []string) error { + + kernels, err := kernel.ListKernels() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + nconfig, _ := node.New() nodes, _ := nconfig.FindAllNodes() nodemap := make(map[string]int) @@ -21,31 +26,53 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodemap[n.KernelVersion.Get()]++ } - images, _ := ioutil.ReadDir(config.KernelParentDir()) + fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") + for _, k := range kernels { + image := kernel.KernelImage(k) - fmt.Printf("%-38s %-16s %-16s %s\n", "KERNEL VERSION", "KERNEL SIZE(k)", "KMODS SIZE(k)", "NODES") - fmt.Println(strings.Repeat("=", 80)) + if nodemap[k] == 0 { + nodemap[k] = 0 + } + fmt.Printf("%-35s %-6t %-6d\n", k, util.IsFile(image), nodemap[k]) - for _, file := range images { - if util.IsDir(path.Join(config.KernelParentDir(), file.Name())) { - var kernel_size int64 - var kmods_size int64 - if util.IsFile(config.KernelImage(file.Name())) { - s, _ := os.Stat(config.KernelImage(file.Name())) - kernel_size = s.Size() / 1024 - } - if util.IsFile(config.KmodsImage(file.Name())) { - s, _ := os.Stat(config.KmodsImage(file.Name())) - kmods_size = s.Size() / 1024 - } + } - if nodemap[file.Name()] > 0 { - fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, nodemap[file.Name()]) - } else { - fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, 0) + /* + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.KernelVersion.Get()]++ + } + + images, _ := ioutil.ReadDir(config.KernelParentDir()) + + fmt.Printf("%-38s %-16s %-16s %s\n", "KERNEL VERSION", "KERNEL SIZE(k)", "KMODS SIZE(k)", "NODES") + fmt.Println(strings.Repeat("=", 80)) + + for _, file := range images { + if util.IsDir(path.Join(config.KernelParentDir(), file.Name())) { + var kernel_size int64 + var kmods_size int64 + if util.IsFile(config.KernelImage(file.Name())) { + s, _ := os.Stat(config.KernelImage(file.Name())) + kernel_size = s.Size() / 1024 + } + if util.IsFile(config.KmodsImage(file.Name())) { + s, _ := os.Stat(config.KmodsImage(file.Name())) + kmods_size = s.Size() / 1024 + } + + if nodemap[file.Name()] > 0 { + fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, nodemap[file.Name()]) + } else { + fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, 0) + } } } - } + + */ return nil } diff --git a/internal/app/wwctl/node/set/main.go b/internal/app/wwctl/node/set/main.go index cfc40362..37b94e39 100644 --- a/internal/app/wwctl/node/set/main.go +++ b/internal/app/wwctl/node/set/main.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -43,6 +44,23 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } + if SetVnfs != "" { + if vnfs.ValidSource(SetVnfs) == true { + imageFile := vnfs.ImageFile(SetVnfs) + if util.IsFile(imageFile) == false { + wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } else { + wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } + for _, n := range nodes { wwlog.Printf(wwlog.VERBOSE, "Evaluating node: %s\n", n.Id.Get()) diff --git a/internal/app/wwctl/node/set/root.go b/internal/app/wwctl/node/set/root.go index d905accf..aa21b77c 100644 --- a/internal/app/wwctl/node/set/root.go +++ b/internal/app/wwctl/node/set/root.go @@ -30,6 +30,7 @@ var ( SetYes bool SetAddProfile []string SetDelProfile []string + SetForce bool ) func init() { @@ -57,6 +58,7 @@ func init() { baseCmd.PersistentFlags().BoolVarP(&SetNodeAll, "all", "a", false, "Set all nodes") baseCmd.PersistentFlags().BoolVarP(&SetYes, "yes", "y", false, "Set 'yes' to all questions asked") + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force configuration (even on error)") } diff --git a/internal/app/wwctl/profile/set/main.go b/internal/app/wwctl/profile/set/main.go index cc185f41..f4058894 100644 --- a/internal/app/wwctl/profile/set/main.go +++ b/internal/app/wwctl/profile/set/main.go @@ -3,6 +3,8 @@ package set import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -46,6 +48,23 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } + if SetVnfs != "" { + if vnfs.ValidSource(SetVnfs) == true { + imageFile := vnfs.ImageFile(SetVnfs) + if util.IsFile(imageFile) == false { + wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } else { + wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } + for _, p := range profiles { wwlog.Printf(wwlog.VERBOSE, "Modifying profile: %s\n", p.Id.Get()) diff --git a/internal/app/wwctl/profile/set/root.go b/internal/app/wwctl/profile/set/root.go index 14e3269e..d841534c 100644 --- a/internal/app/wwctl/profile/set/root.go +++ b/internal/app/wwctl/profile/set/root.go @@ -10,6 +10,7 @@ var ( RunE: CobraRunE, } SetAll bool + SetForce bool SetComment string SetVnfs string SetKernel string @@ -48,6 +49,7 @@ func init() { baseCmd.PersistentFlags().BoolVar(&SetNetDevDel, "netdel", false, "Delete the node's network device") baseCmd.PersistentFlags().BoolVarP(&SetAll, "all", "a", false, "Set all profiles") + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force configuration (even on error)") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/ready/main.go b/internal/app/wwctl/ready/main.go index af7e7ef0..1369b7fc 100644 --- a/internal/app/wwctl/ready/main.go +++ b/internal/app/wwctl/ready/main.go @@ -3,6 +3,7 @@ package ready import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" @@ -36,12 +37,13 @@ func CobraRunE(cmd *cobra.Command, args []string) error { status := true if node.Vnfs.Get() != "" { - v, _ := vnfs.Load(node.Vnfs.Get()) - if util.IsFile(v.Image) == true { + vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) + + if util.IsFile(vnfsImage) == true { vnfs_good = true } else { status = false - wwlog.Printf(wwlog.VERBOSE, "VNFS not found: %s, %s\n", node.Id.Get(), v.Source) + wwlog.Printf(wwlog.VERBOSE, "VNFS not found: %s, %s\n", node.Id.Get(), vnfsImage) } } else { status = false @@ -49,13 +51,13 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if node.KernelVersion.Get() != "" { - if util.IsFile(config.KernelImage(node.KernelVersion.Get())) == true { + if util.IsFile(kernel.KernelImage(node.KernelVersion.Get())) == true { kernel_good = true } else { status = false wwlog.Printf(wwlog.VERBOSE, "Node Kernel not found: %s, %s\n", node.Id.Get(), node.KernelVersion.Get()) } - if util.IsFile(config.KmodsImage(node.KernelVersion.Get())) == true { + if util.IsFile(kernel.KmodsImage(node.KernelVersion.Get())) == true { kmods_good = true } else { status = false diff --git a/internal/app/wwctl/vnfs/build/main.go b/internal/app/wwctl/vnfs/build/main.go index 5952aba4..fa2330e1 100644 --- a/internal/app/wwctl/vnfs/build/main.go +++ b/internal/app/wwctl/vnfs/build/main.go @@ -1,8 +1,6 @@ package build import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" @@ -10,59 +8,72 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var nodes []node.NodeInfo - set := make(map[string]int) - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - if len(args) == 1 && ByNode == true { - var err error - nodes, err = n.SearchByName(args[0]) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + for _, arg := range args { + if vnfs.ValidSource(arg) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name does not exist: %s\n", arg) os.Exit(1) } - for _, node := range nodes { - if node.Vnfs.Defined() == true { - set[node.Vnfs.Get()]++ + vnfs.Build(arg, BuildForce) + } + + /* + var nodes []node.NodeInfo + set := make(map[string]int) + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) == 1 && ByNode == true { + var err error + nodes, err = n.SearchByName(args[0]) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + set[node.Vnfs.Get()]++ + } + } + + } else if BuildAll == true { + var err error + nodes, err = n.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) + set[node.Vnfs.Get()]++ + } + } + + } else if len(args) == 1 { + set[args[0]]++ + } else { + cmd.Usage() + os.Exit(1) + } + + for v := range set { + fmt.Printf("Building VNFS: %s\n", v) + err := vnfs.Build(v, BuildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) } } - } else if BuildAll == true { - var err error - nodes, err = n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) - os.Exit(1) - } - - for _, node := range nodes { - if node.Vnfs.Defined() == true { - wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) - set[node.Vnfs.Get()]++ - } - } - - } else if len(args) == 1 { - set[args[0]]++ - } else { - cmd.Usage() - os.Exit(1) - } - - for v := range set { - fmt.Printf("Building VNFS: %s\n", v) - err := vnfs.Build(v, BuildForce) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } + */ return nil } diff --git a/internal/app/wwctl/vnfs/list/main.go b/internal/app/wwctl/vnfs/list/main.go index 212fcbdf..2499fbac 100644 --- a/internal/app/wwctl/vnfs/list/main.go +++ b/internal/app/wwctl/vnfs/list/main.go @@ -2,18 +2,22 @@ package list import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" - "io/ioutil" "os" - "path" - "strings" ) func CobraRunE(cmd *cobra.Command, args []string) error { + + sources, err := vnfs.ListSources() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + nconfig, _ := node.New() nodes, _ := nconfig.FindAllNodes() nodemap := make(map[string]int) @@ -22,42 +26,64 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodemap[n.Vnfs.Get()]++ } - images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) - - fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") - fmt.Println(strings.Repeat("=", 80)) - - for _, file := range images { - v, err := vnfs.Load(file.Name()) - if err == nil { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[v.Source] > 0 { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) - } else { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) - } + fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") + for _, source := range sources { + image := vnfs.ImageFile(source) + if nodemap[source] == 0 { + nodemap[source] = 0 } - continue + fmt.Printf("%-35s %-6t %-6d\n", source, util.IsFile(image), nodemap[source]) - if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[file.Name()] > 0 { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) - } else { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) - } - } } + + /* + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.Vnfs.Get()]++ + } + + images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) + + fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") + fmt.Println(strings.Repeat("=", 80)) + + for _, file := range images { + v, err := vnfs.Load(file.Name()) + if err == nil { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[v.Source] > 0 { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) + } else { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) + } + + } + continue + + if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[file.Name()] > 0 { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) + } else { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) + } + } + } + */ + return nil } diff --git a/internal/app/wwctl/vnfs/list/root.go b/internal/app/wwctl/vnfs/list/root.go index 58662f4d..80216bb9 100644 --- a/internal/app/wwctl/vnfs/list/root.go +++ b/internal/app/wwctl/vnfs/list/root.go @@ -4,13 +4,13 @@ import "github.com/spf13/cobra" var ( baseCmd = &cobra.Command{ - Use: "list", - Short: "List VNFS images", - Long: "List VNFS images ", - RunE: CobraRunE, + Use: "list", + Short: "List VNFS images", + Long: "List VNFS images ", + RunE: CobraRunE, } SystemOverlay bool - BuildAll bool + BuildAll bool ) func init() { diff --git a/internal/app/wwctl/vnfs/pull/main.go b/internal/app/wwctl/vnfs/pull/main.go new file mode 100644 index 00000000..b85466fe --- /dev/null +++ b/internal/app/wwctl/vnfs/pull/main.go @@ -0,0 +1,54 @@ +package pull + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" + "path" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var name string + uri := args[0] + + if len(args) == 2 { + name = args[1] + } else { + name = path.Base(uri) + fmt.Printf("Setting VNFS name: %s\n", name) + } + + if vnfs.ValidName(name) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", name) + os.Exit(1) + } + + fullPath := vnfs.SourceDir(name) + + if util.IsDir(fullPath) == true { + if SetForce == true { + wwlog.Printf(wwlog.WARN, "Overwriting existing VNFS\n") + err := os.RemoveAll(fullPath) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } else if SetUpdate == true { + wwlog.Printf(wwlog.WARN, "Updating existing VNFS\n") + } else { + wwlog.Printf(wwlog.ERROR, "VNFS Name exists, specify --force, --update, or choose a different name: %s\n", name) + os.Exit(1) + } + } + + err := vnfs.PullURI(uri, name) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not pull image: %s\n", err) + os.Exit(1) + } + + return nil +} diff --git a/internal/app/wwctl/vnfs/pull/root.go b/internal/app/wwctl/vnfs/pull/root.go new file mode 100644 index 00000000..93d88329 --- /dev/null +++ b/internal/app/wwctl/vnfs/pull/root.go @@ -0,0 +1,26 @@ +package pull + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "pull", + Short: "Pull Source OCI VNFS images", + Long: "Pull Source OCI VNFS images ", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), + } + SetForce bool + SetUpdate bool +) + +func init() { + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force overwrite of an existing VNFS") + baseCmd.PersistentFlags().BoolVarP(&SetUpdate, "update", "u", false, "Update and overwrite an existing VNFS") + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/vnfs/root.go b/internal/app/wwctl/vnfs/root.go index 851d1d2b..0b10b5c2 100644 --- a/internal/app/wwctl/vnfs/root.go +++ b/internal/app/wwctl/vnfs/root.go @@ -3,14 +3,15 @@ package vnfs import ( "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/build" "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/list" + "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/pull" "github.com/spf13/cobra" ) var ( baseCmd = &cobra.Command{ - Use: "vnfs", - Short: "VNFS image management", - Long: "Virtual Node File System (VNFS) image management", + Use: "vnfs", + Short: "VNFS image management", + Long: "Virtual Node File System (VNFS) image management", } test bool ) @@ -18,10 +19,11 @@ var ( func init() { baseCmd.AddCommand(build.GetCommand()) baseCmd.AddCommand(list.GetCommand()) + baseCmd.AddCommand(pull.GetCommand()) + } // GetRootCommand returns the root cobra.Command for the application. func GetCommand() *cobra.Command { return baseCmd } - diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 8586e396..5a189525 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -11,10 +11,6 @@ const ( LocalStateDir = "/var/warewulf" ) -func NodeConfig() string { - return fmt.Sprintf("%s/nodes.conf", LocalStateDir) -} - func OverlayDir() string { return fmt.Sprintf("%s/overlays/", LocalStateDir) } @@ -27,25 +23,13 @@ func RuntimeOverlayDir() string { return path.Join(OverlayDir(), "/runtime") } -func VnfsImageParentDir() string { - return fmt.Sprintf("%s/provision/vnfs/", LocalStateDir) -} - -func VnfsChrootParentDir() string { - return fmt.Sprintf("%s/chroot/", LocalStateDir) -} - -func KernelParentDir() string { - return fmt.Sprintf("%s/provision/kernel/", LocalStateDir) -} - func SystemOverlaySource(overlayName string) string { if overlayName == "" { wwlog.Printf(wwlog.ERROR, "System overlay name is not defined\n") return "" } - if util.TaintCheck(overlayName, "^[a-zA-Z0-9-._]+$") == false { + if util.ValidString(overlayName, "^[a-zA-Z0-9-._]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", overlayName) return "" } @@ -59,7 +43,7 @@ func RuntimeOverlaySource(overlayName string) string { return "" } - if util.TaintCheck(overlayName, "^[a-zA-Z0-9-._]+$") == false { + if util.ValidString(overlayName, "^[a-zA-Z0-9-._]+$") == false { wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", overlayName) return "" } @@ -67,41 +51,13 @@ func RuntimeOverlaySource(overlayName string) string { return path.Join(RuntimeOverlayDir(), overlayName) } -func KernelImage(kernelVersion string) string { - if kernelVersion == "" { - wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") - return "" - } - - if util.TaintCheck(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { - wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) - return "" - } - - return path.Join(KernelParentDir(), kernelVersion, "vmlinuz") -} - -func KmodsImage(kernelVersion string) string { - if kernelVersion == "" { - wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") - return "" - } - - if util.TaintCheck(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { - wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) - return "" - } - - return path.Join(KernelParentDir(), kernelVersion, "kmods.img") -} - func SystemOverlayImage(nodeName string) string { if nodeName == "" { wwlog.Printf(wwlog.ERROR, "Node name is not defined\n") return "" } - if util.TaintCheck(nodeName, "^[a-zA-Z0-9-._:]+$") == false { + if util.ValidString(nodeName, "^[a-zA-Z0-9-._:]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", nodeName) return "" } @@ -115,42 +71,10 @@ func RuntimeOverlayImage(nodeName string) string { return "" } - if util.TaintCheck(nodeName, "^[a-zA-Z0-9-._:]+$") == false { + if util.ValidString(nodeName, "^[a-zA-Z0-9-._:]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", nodeName) return "" } return fmt.Sprintf("%s/provision/overlays/runtime/%s.img", LocalStateDir, nodeName) } - -func VnfsImageDir(uri string) string { - if uri == "" { - wwlog.Printf(wwlog.ERROR, "VNFS URI is not defined\n") - return "" - } - - if util.TaintCheck(uri, "^[a-zA-Z0-9-._:]+$") == false { - wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", uri) - return "" - } - - return path.Join(VnfsImageParentDir(), uri) -} - -func VnfsImage(uri string) string { - return path.Join(VnfsImageDir(uri), "image") -} - -func VnfsChroot(uri string) string { - if uri == "" { - wwlog.Printf(wwlog.ERROR, "VNFS name is not defined\n") - return "" - } - - if util.TaintCheck(uri, "^[a-zA-Z0-9-._:]+$") == false { - wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", uri) - return "" - } - - return path.Join(VnfsChrootParentDir(), uri) -} diff --git a/internal/pkg/kernel/kernel.go b/internal/pkg/kernel/kernel.go index 26109458..e59444f7 100644 --- a/internal/pkg/kernel/kernel.go +++ b/internal/pkg/kernel/kernel.go @@ -6,17 +6,75 @@ import ( "github.com/hpcng/warewulf/internal/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" + "io/ioutil" "os" "os/exec" "path" ) +func ParentDir() string { + return path.Join(config.LocalStateDir, "provision/kernel") +} + +func KernelImage(kernelVersion string) string { + if kernelVersion == "" { + wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") + return "" + } + + if util.ValidString(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { + wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) + return "" + } + + return path.Join(ParentDir(), kernelVersion, "vmlinuz") +} + +func KmodsImage(kernelVersion string) string { + if kernelVersion == "" { + wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") + return "" + } + + if util.ValidString(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { + wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) + return "" + } + + return path.Join(ParentDir(), kernelVersion, "kmods.img") +} + +func ListKernels() ([]string, error) { + var ret []string + + err := os.MkdirAll(ParentDir(), 0755) + if err != nil { + return ret, errors.New("Could not create Kernel parent directory: " + ParentDir()) + } + + wwlog.Printf(wwlog.DEBUG, "Searching for Kernel image directories: %s\n", ParentDir()) + + kernels, err := ioutil.ReadDir(ParentDir()) + if err != nil { + return ret, err + } + + for _, kernel := range kernels { + wwlog.Printf(wwlog.VERBOSE, "Found Kernel: %s\n", kernel.Name()) + + ret = append(ret, kernel.Name()) + + } + + return ret, nil +} + func Build(kernelVersion string) error { kernelImage := "/boot/vmlinuz-" + kernelVersion kernelDrivers := "/lib/modules/" + kernelVersion - kernelDestination := config.KernelImage(kernelVersion) - driversDestination := config.KmodsImage(kernelVersion) + kernelDestination := KernelImage(kernelVersion) + driversDestination := KmodsImage(kernelVersion) // Create the destination paths just in case it doesn't exist os.MkdirAll(path.Dir(kernelDestination), 0755) @@ -30,6 +88,7 @@ func Build(kernelVersion string) error { return errors.New("Could not locate kernel drivers: " + kernelDrivers) } + wwlog.Printf(wwlog.VERBOSE, "Setting up Kernel\n") if _, err := os.Stat(kernelImage); err == nil { err := util.CopyFile(kernelImage, kernelDestination) if err != nil { @@ -38,6 +97,7 @@ func Build(kernelVersion string) error { wwlog.Printf(wwlog.INFO, "%-45s: Done\n", "vmlinuz-"+kernelVersion) } + wwlog.Printf(wwlog.VERBOSE, "Building Kernel driver image\n") if _, err := os.Stat(kernelDrivers); err == nil { cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelDrivers, driversDestination) err := exec.Command("/bin/sh", "-c", cmd).Run() diff --git a/internal/pkg/oci/cache.go b/internal/pkg/oci/cache.go index 6aaa0828..369efaf4 100644 --- a/internal/pkg/oci/cache.go +++ b/internal/pkg/oci/cache.go @@ -79,15 +79,15 @@ func NewCache(opts ...CacheOpt) (*Cache, error) { } func (c *Cache) Pull(ctx context.Context, uri string, sysCtx *types.SystemContext) (string, error) { - p, err := newPuller( - optSetBlobCachePath(c.blobDir()), - optSetSystemContext(sysCtx), + p, err := NewPuller( + OptSetBlobCachePath(c.blobDir()), + OptSetSystemContext(sysCtx), ) if err != nil { return "", err } - id, err := p.generateID(ctx, uri) + id, err := p.GenerateID(ctx, uri) if err != nil { return "", err } @@ -107,7 +107,7 @@ func (c *Cache) Pull(ctx context.Context, uri string, sysCtx *types.SystemContex } // populate entry - if err := p.pull(ctx, uri, path); err != nil { + if err := p.Pull(ctx, uri, path); err != nil { // clean up entry on error os.RemoveAll(path) return "", err diff --git a/internal/pkg/oci/puller.go b/internal/pkg/oci/puller.go index 8b497ebf..db393024 100644 --- a/internal/pkg/oci/puller.go +++ b/internal/pkg/oci/puller.go @@ -23,21 +23,21 @@ import ( type pullerOpt func(*puller) error -func optSetBlobCachePath(path string) pullerOpt { +func OptSetBlobCachePath(path string) pullerOpt { return func(p *puller) error { p.blobCachePath = path return nil } } -func optSetTmpDirPath(path string) pullerOpt { +func OptSetTmpDirPath(path string) pullerOpt { return func(p *puller) error { p.tmpDirPath = path return nil } } -func optSetSystemContext(s *types.SystemContext) pullerOpt { +func OptSetSystemContext(s *types.SystemContext) pullerOpt { return func(p *puller) error { p.sysCtx = s return nil @@ -51,7 +51,7 @@ type puller struct { sysCtx *types.SystemContext } -func newPuller(opts ...pullerOpt) (*puller, error) { +func NewPuller(opts ...pullerOpt) (*puller, error) { p := &puller{ // default to a sensible value, but caller should set this with opts blobCachePath: filepath.Join(defaultCachePath, blobPrefix), @@ -93,8 +93,8 @@ func getReference(uri string) (types.ImageReference, error) { } } -// generateID stores and returns a unique identifier derived from the sha256sum of the image manifest -func (p *puller) generateID(ctx context.Context, uri string) (string, error) { +// GenerateID stores and returns a unique identifier derived from the sha256sum of the image manifest +func (p *puller) GenerateID(ctx context.Context, uri string) (string, error) { ref, err := getReference(uri) if err != nil { return "", fmt.Errorf("unable to parse uri: %v", err) @@ -114,7 +114,7 @@ func (p *puller) generateID(ctx context.Context, uri string) (string, error) { return p.id, nil } -func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { +func (p *puller) Pull(ctx context.Context, uri, dst string) (err error) { srcRef, err := getReference(uri) if err != nil { return fmt.Errorf("unable to parse uri: %v", err) @@ -135,7 +135,7 @@ func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { // copy to cache location _, err = copy.Image(ctx, policyCtx, cacheRef, srcRef, ©.Options{ ReportWriter: os.Stdout, - SourceCtx: p.sysCtx, + SourceCtx: p.sysCtx, }) if err != nil { return err diff --git a/internal/pkg/overlay/funcmap.go b/internal/pkg/overlay/funcmap.go index 3cf8e0b1..7843a4ae 100644 --- a/internal/pkg/overlay/funcmap.go +++ b/internal/pkg/overlay/funcmap.go @@ -1,7 +1,6 @@ package overlay import ( - "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "io/ioutil" @@ -26,14 +25,14 @@ func templateVnfsFileInclude(vnfsname string, filepath string) string { return "" } - v, _ := vnfs.Load(vnfsname) - vnfsDir := v.Chroot - - if util.IsDir(vnfsDir) == false { - wwlog.Printf(wwlog.WARN, "Template requires VNFS (%s) to be imported: %s\n", vnfsname, filepath) + if vnfs.ValidSource(vnfsname) == false { + wwlog.Printf(wwlog.WARN, "Template required VNFS does not exist: %s\n", vnfsname) return "" } - wwlog.Printf(wwlog.DEBUG, "IncludeVnfs file from: %s/%s\n", vnfsDir, filepath) + + vnfsDir := vnfs.RootFsDir(vnfsname) + + wwlog.Printf(wwlog.DEBUG, "Including file from VNFS: %s:%s\n", vnfsDir, filepath) content, err := ioutil.ReadFile(path.Join(vnfsDir, filepath)) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index a768e04e..c342afd2 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -114,7 +114,7 @@ func IsFile(path string) bool { return false } -func TaintCheck(pattern string, expr string) bool { +func ValidString(pattern string, expr string) bool { if b, _ := regexp.MatchString(expr, pattern); b == true { return true } @@ -122,7 +122,7 @@ func TaintCheck(pattern string, expr string) bool { } func ValidateOrDie(message string, pattern string, expr string) { - if TaintCheck(pattern, expr) == false { + if ValidString(pattern, expr) == false { wwlog.Printf(wwlog.ERROR, "%s does not validate: '%s'\n", message, pattern) os.Exit(1) } diff --git a/internal/pkg/vnfs/build.go b/internal/pkg/vnfs/build.go new file mode 100644 index 00000000..101d6b7c --- /dev/null +++ b/internal/pkg/vnfs/build.go @@ -0,0 +1,54 @@ +package vnfs + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "os" + "os/exec" + "path" +) + +func Build(name string, buildForce bool) { + + rootfsPath := RootFsDir(name) + imagePath := ImageFile(name) + + if ValidSource(name) == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping (bad path)\n", name) + return + } + + if buildForce == false { + wwlog.Printf(wwlog.DEBUG, "Checking if there have been any updates to the VNFS directory\n") + if util.PathIsNewer(rootfsPath, imagePath) { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", name) + return + } + } + + wwlog.Printf(wwlog.DEBUG, "Making parent directory for: %s\n", name) + err := os.MkdirAll(path.Dir(imagePath), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + + wwlog.Printf(wwlog.DEBUG, "Making parent directory for: %s\n", rootfsPath) + err = os.MkdirAll(path.Dir(rootfsPath), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + + wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", rootfsPath, imagePath) + cmd := fmt.Sprintf("cd %s; find . | cpio --quiet -o -H newc | gzip -c > \"%s\"", rootfsPath, imagePath) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Failed building VNFS: %s\n", err) + return + } + + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", name) + +} diff --git a/internal/pkg/vnfs/pull.go b/internal/pkg/vnfs/pull.go new file mode 100644 index 00000000..f9fd75aa --- /dev/null +++ b/internal/pkg/vnfs/pull.go @@ -0,0 +1,45 @@ +package vnfs + +import ( + "context" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/errors" + "github.com/hpcng/warewulf/internal/pkg/oci" + "os" +) + +func PullURI(uri string, name string) error { + OciBlobCacheDir := config.LocalStateDir + "/oci/blobs" + + err := os.MkdirAll(OciBlobCacheDir, 0755) + if err != nil { + return err + } + + if ValidName(name) == false { + return errors.New("VNFS name contains illegal characters: " + name) + } + + fullPath := RootFsDir(name) + + err = os.MkdirAll(fullPath, 0755) + if err != nil { + return err + } + + p, err := oci.NewPuller( + oci.OptSetBlobCachePath(OciBlobCacheDir), + oci.OptSetSystemContext(nil), + ) + if err != nil { + return err + } + + p.GenerateID(context.Background(), uri) + + if err := p.Pull(context.Background(), uri, fullPath); err != nil { + return err + } + + return nil +} diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 32454fb9..ac40fc9e 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -5,169 +5,86 @@ import ( "github.com/hpcng/warewulf/internal/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "gopkg.in/yaml.v2" "io/ioutil" "os" "path" - "strings" ) -type VnfsObject struct { - Name string - Source string - Chroot string - Image string - Config string +func ValidName(name string) bool { + if util.ValidString(name, "^[a-zA-Z0-9.:-]+$") == false { + wwlog.Printf(wwlog.WARN, "VNFS name has illegal characters: %s\n", name) + return false + } + return true } -func Load(name string) (VnfsObject, error) { - var ret VnfsObject +func SourceParentDir() string { + return path.Join(config.LocalStateDir, "chroots") +} - if name == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") - } +func SourceDir(name string) string { + return path.Join(SourceParentDir(), name) +} - pathFriendlyName := CleanName(name) +func RootFsDir(name string) string { + return path.Join(SourceDir(name), "rootfs") +} - configFile := path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") +func ImageParentDir() string { + return path.Join(config.LocalStateDir, "provision/vnfs/") +} - if util.IsFile(configFile) == false { - return ret, errors.New("VNFS has not been imported: " + name) - } +func ImageFile(name string) string { + return path.Join(ImageParentDir(), name+".img.gz") +} - data, err := ioutil.ReadFile(configFile) +func ListSources() ([]string, error) { + var ret []string + + err := os.MkdirAll(SourceParentDir(), 0755) if err != nil { - return ret, errors.New("Error reading VNFS configuration file: " + name) + return ret, errors.New("Could not create VNFS source parent directory: " + SourceParentDir()) } + wwlog.Printf(wwlog.DEBUG, "Searching for VNFS Rootfs directories: %s\n", SourceParentDir()) - err = yaml.Unmarshal(data, &ret) + sources, err := ioutil.ReadDir(SourceParentDir()) if err != nil { return ret, err } - return ret, nil -} + for _, source := range sources { + wwlog.Printf(wwlog.VERBOSE, "Found VNFS source: %s\n", source.Name()) -func CleanName(source string) string { - var tmp string - - if strings.HasPrefix(source, "/") == true { - tmp = path.Base(source) - } else { - tmp = source - } - - tmp = strings.ReplaceAll(tmp, "://", ":") - tmp = strings.ReplaceAll(tmp, "/", "-") - tmp = strings.ReplaceAll(tmp, ":", ":") - - return tmp -} - -func New(source string) (VnfsObject, error) { - var ret VnfsObject - - if source == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") - } - - if util.IsFile(ret.Config) { - return Load(source) - } - - pathFriendlyName := CleanName(source) - - if strings.HasPrefix(source, "/") == true { - ret.Source = source - ret.Name = pathFriendlyName - } else { - tmp := strings.ReplaceAll(source, "://", "-") - tmp = strings.ReplaceAll(tmp, "/", ".") - tmp = strings.ReplaceAll(tmp, ":", ".") - ret.Name = source - ret.Source = source - } - - ret.Chroot = config.VnfsChroot(pathFriendlyName) - ret.Image = config.VnfsImage(pathFriendlyName) - ret.Config = path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") - - return ret, nil -} - -func (self *VnfsObject) SaveConfig() error { - - out, err := yaml.Marshal(self) - if err != nil { - return err - } - - file, err := os.OpenFile(self.Config, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - - defer file.Close() - - _, err = file.WriteString(string(out)) - if err != nil { - return err - } - - return nil -} - -func Build(name string, force bool) error { - - vnfs, err := New(name) - if err != nil { - return err - } - - wwlog.Printf(wwlog.VERBOSE, "Building VNFS: %s\n", vnfs.Name) - if strings.HasPrefix(vnfs.Source, "/") { - if strings.HasSuffix(vnfs.Source, "tar.gz") { - //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) - wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", vnfs.Name) - } else { - BuildContainerdir(vnfs, force) + if ValidName(source.Name()) == false { + continue } - } else { - BuildDocker(vnfs, force) + + if ValidSource(source.Name()) == false { + continue + } + + ret = append(ret, source.Name()) } - err = vnfs.SaveConfig() - if err != nil { - return err - } - - return nil + return ret, nil } -func (self *VnfsObject) Nameold() string { - if self.Source == "" { - return "" +func ValidSource(name string) bool { + fullPath := RootFsDir(name) + + if ValidName(name) == false { + return false } - if strings.HasPrefix(self.Source, "/") { - return path.Base(self.Source) + if util.IsDir(fullPath) == false { + wwlog.Printf(wwlog.VERBOSE, "Location is not a VNFS source directory: %s\n", name) + return false } - return self.Source -} - -func NameClean1(SourcePath string) string { - if SourcePath == "" { - return "" - } - - if strings.HasPrefix(SourcePath, "/") { - return path.Base(SourcePath) - } - uri := strings.Split(SourcePath, "://") - - return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") + if util.IsFile(path.Join(fullPath, "/sbin/init")) == false { + wwlog.Printf(wwlog.VERBOSE, "VNFS Source does not have a valid /sbin/init: %s\n", name) + return false + } + + return true } diff --git a/internal/pkg/vnfs/containerdir.go b/internal/pkg/vnfs_old/containerdir.go similarity index 100% rename from internal/pkg/vnfs/containerdir.go rename to internal/pkg/vnfs_old/containerdir.go diff --git a/internal/pkg/vnfs/docker.go b/internal/pkg/vnfs_old/docker.go similarity index 100% rename from internal/pkg/vnfs/docker.go rename to internal/pkg/vnfs_old/docker.go diff --git a/internal/pkg/vnfs/util.go b/internal/pkg/vnfs_old/util.go similarity index 100% rename from internal/pkg/vnfs/util.go rename to internal/pkg/vnfs_old/util.go diff --git a/internal/pkg/vnfs_old/vnfs.go b/internal/pkg/vnfs_old/vnfs.go new file mode 100644 index 00000000..32454fb9 --- /dev/null +++ b/internal/pkg/vnfs_old/vnfs.go @@ -0,0 +1,173 @@ +package vnfs + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/errors" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "gopkg.in/yaml.v2" + "io/ioutil" + "os" + "path" + "strings" +) + +type VnfsObject struct { + Name string + Source string + Chroot string + Image string + Config string +} + +func Load(name string) (VnfsObject, error) { + var ret VnfsObject + + if name == "" { + wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") + return ret, errors.New("Called vnfs.Load() without a VNFS name") + } + + pathFriendlyName := CleanName(name) + + configFile := path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") + + if util.IsFile(configFile) == false { + return ret, errors.New("VNFS has not been imported: " + name) + } + + data, err := ioutil.ReadFile(configFile) + if err != nil { + return ret, errors.New("Error reading VNFS configuration file: " + name) + } + + err = yaml.Unmarshal(data, &ret) + if err != nil { + return ret, err + } + + return ret, nil +} + +func CleanName(source string) string { + var tmp string + + if strings.HasPrefix(source, "/") == true { + tmp = path.Base(source) + } else { + tmp = source + } + + tmp = strings.ReplaceAll(tmp, "://", ":") + tmp = strings.ReplaceAll(tmp, "/", "-") + tmp = strings.ReplaceAll(tmp, ":", ":") + + return tmp +} + +func New(source string) (VnfsObject, error) { + var ret VnfsObject + + if source == "" { + wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") + return ret, errors.New("Called vnfs.Load() without a VNFS name") + } + + if util.IsFile(ret.Config) { + return Load(source) + } + + pathFriendlyName := CleanName(source) + + if strings.HasPrefix(source, "/") == true { + ret.Source = source + ret.Name = pathFriendlyName + } else { + tmp := strings.ReplaceAll(source, "://", "-") + tmp = strings.ReplaceAll(tmp, "/", ".") + tmp = strings.ReplaceAll(tmp, ":", ".") + ret.Name = source + ret.Source = source + } + + ret.Chroot = config.VnfsChroot(pathFriendlyName) + ret.Image = config.VnfsImage(pathFriendlyName) + ret.Config = path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") + + return ret, nil +} + +func (self *VnfsObject) SaveConfig() error { + + out, err := yaml.Marshal(self) + if err != nil { + return err + } + + file, err := os.OpenFile(self.Config, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + + defer file.Close() + + _, err = file.WriteString(string(out)) + if err != nil { + return err + } + + return nil +} + +func Build(name string, force bool) error { + + vnfs, err := New(name) + if err != nil { + return err + } + + wwlog.Printf(wwlog.VERBOSE, "Building VNFS: %s\n", vnfs.Name) + if strings.HasPrefix(vnfs.Source, "/") { + if strings.HasSuffix(vnfs.Source, "tar.gz") { + //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) + wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", vnfs.Name) + } else { + BuildContainerdir(vnfs, force) + } + } else { + BuildDocker(vnfs, force) + } + + err = vnfs.SaveConfig() + if err != nil { + return err + } + + return nil +} + +func (self *VnfsObject) Nameold() string { + if self.Source == "" { + return "" + } + + if strings.HasPrefix(self.Source, "/") { + return path.Base(self.Source) + } + + return self.Source +} + +func NameClean1(SourcePath string) string { + if SourcePath == "" { + return "" + } + + if strings.HasPrefix(SourcePath, "/") { + return path.Base(SourcePath) + } + uri := strings.Split(SourcePath, "://") + + return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") +} From 8c11c2b304a5695d2e424cbc81da125665049c75 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 00:47:00 -0800 Subject: [PATCH 2/9] Renamed `vnfs` to `container`, finished VNFS and kernel refactoring --- Makefile | 8 +- etc/ipxe/default.ipxe | 7 +- etc/nodes.conf | 2 +- etc/warewulf.conf | 4 +- internal/app/warewulfd/response/ipxe.go | 20 +++-- internal/app/warewulfd/response/kernel.go | 4 +- internal/app/warewulfd/response/kmods.go | 4 +- internal/app/warewulfd/response/vnfs.go | 34 ------- internal/app/warewulfd/warewulfd.go | 2 +- internal/app/wwctl/build/main.go | 4 +- internal/app/wwctl/build/root.go | 24 ++--- internal/app/wwctl/node/add/root.go | 2 +- internal/app/wwctl/node/list/main.go | 6 +- internal/app/wwctl/node/set/main.go | 18 ++-- internal/app/wwctl/node/set/root.go | 8 +- internal/app/wwctl/profile/list/main.go | 2 +- internal/app/wwctl/profile/set/main.go | 18 ++-- internal/app/wwctl/profile/set/root.go | 8 +- internal/app/wwctl/ready/main.go | 6 +- internal/app/wwctl/root.go | 4 +- internal/app/wwctl/vnfs/build/main.go | 79 ---------------- internal/app/wwctl/vnfs/build/root.go | 28 ------ internal/app/wwctl/vnfs/list/main.go | 89 ------------------- internal/app/wwctl/vnfs/list/root.go | 25 ------ internal/app/wwctl/vnfs/pull/main.go | 54 ----------- internal/app/wwctl/vnfs/pull/root.go | 26 ------ internal/app/wwctl/vnfs/root.go | 29 ------ internal/pkg/{vnfs => container}/build.go | 2 +- internal/pkg/{vnfs => container}/pull.go | 2 +- internal/pkg/{vnfs => container}/vnfs.go | 4 +- internal/pkg/node/constructors.go | 6 +- internal/pkg/node/datastructure.go | 4 +- internal/pkg/node/modifiers.go | 4 +- internal/pkg/oci/cache.go | 2 +- internal/pkg/overlay/funcmap.go | 20 ++--- internal/pkg/overlay/overlay.go | 54 ++++++----- internal/pkg/vnfs_old/docker.go | 2 +- internal/pkg/vnfs_old/vnfs.go | 8 +- overlays/runtime/default/etc/group.ww | 2 +- overlays/runtime/default/etc/passwd.ww | 2 +- overlays/system/default/etc/hostname.ww | 2 +- .../sysconfig/network-scripts/ifcfg-eth0.ww | 8 +- .../system/default/etc/sysconfig/network.ww | 2 +- overlays/system/default/ipmi.ww | 8 +- 44 files changed, 143 insertions(+), 504 deletions(-) delete mode 100644 internal/app/warewulfd/response/vnfs.go delete mode 100644 internal/app/wwctl/vnfs/build/main.go delete mode 100644 internal/app/wwctl/vnfs/build/root.go delete mode 100644 internal/app/wwctl/vnfs/list/main.go delete mode 100644 internal/app/wwctl/vnfs/list/root.go delete mode 100644 internal/app/wwctl/vnfs/pull/main.go delete mode 100644 internal/app/wwctl/vnfs/pull/root.go delete mode 100644 internal/app/wwctl/vnfs/root.go rename internal/pkg/{vnfs => container}/build.go (98%) rename internal/pkg/{vnfs => container}/pull.go (97%) rename internal/pkg/{vnfs => container}/vnfs.go (95%) diff --git a/Makefile b/Makefile index 0384ddca..4675addf 100644 --- a/Makefile +++ b/Makefile @@ -49,16 +49,16 @@ files: all install -d -m 0755 /etc/warewulf/ipxe install -d -m 0755 /var/lib/tftpboot/warewulf/ipxe/ cp -r etc/* /etc/warewulf/ - cp -r tftpboot/* /var/lib/tftpboot/warewulf/ipxe/ - restorecon -r /var/lib/tftpboot/warewulf +# cp -r tftpboot/* /var/lib/tftpboot/warewulf/ipxe/ +# restorecon -r /var/lib/tftpboot/warewulf cp -r overlays /var/warewulf/ chmod +x /var/warewulf/overlays/system/default/init mkdir -p /var/warewulf/overlays/system/default/warewulf/bin/ cp wwclient /var/warewulf/overlays/system/default/warewulf/bin/ services: files - sudo systemctl enable tftp - sudo systemctl restart tftp +# sudo systemctl enable tftp +# sudo systemctl restart tftp warewulfd: cd cmd/warewulfd; go build -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../warewulfd diff --git a/etc/ipxe/default.ipxe b/etc/ipxe/default.ipxe index 2df1f61c..42daae76 100644 --- a/etc/ipxe/default.ipxe +++ b/etc/ipxe/default.ipxe @@ -4,12 +4,15 @@ echo echo ================================================================================ echo Warewulf v4 now booting: {{.Fqdn}} echo +echo Container: {{.ContainerName}} +echo Kernel: {{.KernelVersion}} +echo KernelArgs: {{.KernelArgs}} echo set base http://{{.Ipaddr}}:{{.Port}} -kernel ${base}/kernel/{{.Hwaddr}} {{.Kernelargs}} || reboot -initrd ${base}/vnfs/{{.Hwaddr}} || reboot +kernel ${base}/kernel/{{.Hwaddr}} {{.KernelArgs}} || reboot +initrd ${base}/container/{{.Hwaddr}} || reboot initrd ${base}/kmods/{{.Hwaddr}} || reboot initrd ${base}/overlay-system/{{.Hwaddr}} || reboot boot || reboot diff --git a/etc/nodes.conf b/etc/nodes.conf index fb24e894..83b14876 100644 --- a/etc/nodes.conf +++ b/etc/nodes.conf @@ -1,7 +1,7 @@ nodeprofiles: default: comment: "This profile is automatically included for each node" - vnfs: "" + container name: "" kernel version: "" kernel args: crashkernel=no quiet diff --git a/etc/warewulf.conf b/etc/warewulf.conf index 380c6d55..eb8c4eba 100644 --- a/etc/warewulf.conf +++ b/etc/warewulf.conf @@ -9,8 +9,8 @@ dhcp: range start: 192.168.1.150 range end: 192.168.1.200 template: default - systemd name: dhcp-server + systemd name: dhcpd tftp: enabled: true tftproot: /var/lib/tftpboot - systemd name: tftp-server + systemd name: tftp diff --git a/internal/app/warewulfd/response/ipxe.go b/internal/app/warewulfd/response/ipxe.go index d4efa341..794159b5 100644 --- a/internal/app/warewulfd/response/ipxe.go +++ b/internal/app/warewulfd/response/ipxe.go @@ -13,13 +13,14 @@ import ( ) type iPxeTemplate struct { - Hostname string - Fqdn string - Vnfs string - Hwaddr string - Ipaddr string - Port string - Kernelargs string + Hostname string + Fqdn string + ContainerName string + Hwaddr string + Ipaddr string + Port string + KernelArgs string + KernelVersion string } func IpxeSend(w http.ResponseWriter, req *http.Request) { @@ -70,8 +71,9 @@ func IpxeSend(w http.ResponseWriter, req *http.Request) { replace.Port = strconv.Itoa(conf.Warewulf.Port) replace.Hostname = node.Id.Get() replace.Hwaddr = url[2] - replace.Vnfs = node.Vnfs.Get() - replace.Kernelargs = node.KernelArgs.Get() + replace.ContainerName = node.ContainerName.Get() + replace.KernelArgs = node.KernelArgs.Get() + replace.KernelVersion = node.KernelVersion.Get() err = tmpl.Execute(w, replace) if err != nil { diff --git a/internal/app/warewulfd/response/kernel.go b/internal/app/warewulfd/response/kernel.go index a7683617..e530de98 100644 --- a/internal/app/warewulfd/response/kernel.go +++ b/internal/app/warewulfd/response/kernel.go @@ -1,7 +1,7 @@ package response import ( - "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "log" "net/http" ) @@ -16,7 +16,7 @@ func KernelSend(w http.ResponseWriter, req *http.Request) { } if node.KernelVersion.Defined() == true { - fileName := config.KernelImage(node.KernelVersion.Get()) + fileName := kernel.KernelImage(node.KernelVersion.Get()) err := sendFile(w, fileName, node.Id.Get()) if err != nil { diff --git a/internal/app/warewulfd/response/kmods.go b/internal/app/warewulfd/response/kmods.go index c4366008..4622b3b2 100644 --- a/internal/app/warewulfd/response/kmods.go +++ b/internal/app/warewulfd/response/kmods.go @@ -1,7 +1,7 @@ package response import ( - "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "log" "net/http" ) @@ -16,7 +16,7 @@ func KmodsSend(w http.ResponseWriter, req *http.Request) { } if node.KernelVersion.Defined() == true { - fileName := config.KmodsImage(node.KernelVersion.Get()) + fileName := kernel.KmodsImage(node.KernelVersion.Get()) err := sendFile(w, fileName, node.Id.Get()) if err != nil { diff --git a/internal/app/warewulfd/response/vnfs.go b/internal/app/warewulfd/response/vnfs.go deleted file mode 100644 index 38ac1c60..00000000 --- a/internal/app/warewulfd/response/vnfs.go +++ /dev/null @@ -1,34 +0,0 @@ -package response - -import ( - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "log" - "net/http" -) - -func VnfsSend(w http.ResponseWriter, req *http.Request) { - - node, err := getSanity(req) - if err != nil { - w.WriteHeader(404) - log.Panicln(err) - return - } - - if node.Vnfs.Defined() == true { - vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) - - err = sendFile(w, vnfsImage, node.Id.Get()) - if err != nil { - log.Printf("ERROR1: %s\n", err) - w.WriteHeader(503) - } else { - log.Printf("SEND: %15s: %s\n", node.Id.Get(), vnfsImage) - } - } else { - w.WriteHeader(503) - log.Printf("ERROR: No VNFS set for node %s\n", node.Id.Get()) - } - - return -} diff --git a/internal/app/warewulfd/warewulfd.go b/internal/app/warewulfd/warewulfd.go index dbdacebe..a6805d2a 100644 --- a/internal/app/warewulfd/warewulfd.go +++ b/internal/app/warewulfd/warewulfd.go @@ -14,7 +14,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { http.HandleFunc("/ipxe/", response.IpxeSend) http.HandleFunc("/kernel/", response.KernelSend) http.HandleFunc("/kmods/", response.KmodsSend) - http.HandleFunc("/vnfs/", response.VnfsSend) + http.HandleFunc("/container/", response.ContainerSend) http.HandleFunc("/overlay-system/", response.SystemOverlaySend) http.HandleFunc("/overlay-runtime", response.RuntimeOverlaySend) diff --git a/internal/app/wwctl/build/main.go b/internal/app/wwctl/build/main.go index 4bb827af..257fa477 100644 --- a/internal/app/wwctl/build/main.go +++ b/internal/app/wwctl/build/main.go @@ -1,9 +1,9 @@ package build import ( + "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" @@ -47,7 +47,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { set[node.Vnfs.Get()]++ } for e := range set { - vnfs.Build(e, buildForce) + container.Build(e, buildForce) } } diff --git a/internal/app/wwctl/build/root.go b/internal/app/wwctl/build/root.go index 32329651..dd8f29cc 100644 --- a/internal/app/wwctl/build/root.go +++ b/internal/app/wwctl/build/root.go @@ -6,24 +6,24 @@ import ( var ( baseCmd = &cobra.Command{ - Use: "build", - Short: "Warewulf build subcommand", - Long: "Warewulf build is used to build VNFS, kernel, and system-overlay objects for\n" + - "provisioning. The default usage will be to build and/or update anything\n" + - "that seems to be needed.", - RunE: CobraRunE, + Use: "build", + Short: "Warewulf build subcommand", + Long: "Warewulf build is used to build VNFS, kernel, and system-overlay objects for\n" + + "provisioning. The default usage will be to build and/or update anything\n" + + "that seems to be needed.", + RunE: CobraRunE, } - buildVnfs bool - buildKernel bool + buildVnfs bool + buildKernel bool buildRuntimeOverlay bool - buildSystemOverlay bool - buildAll bool - buildForce bool + buildSystemOverlay bool + buildAll bool + buildForce bool ) func init() { - baseCmd.PersistentFlags().BoolVarP(&buildVnfs, "vnfs", "V", false, "Build and/or update VNFS images.") + baseCmd.PersistentFlags().BoolVarP(&buildVnfs, "container", "V", false, "Build and/or update VNFS images.") baseCmd.PersistentFlags().BoolVarP(&buildKernel, "kernel", "K", false, "Build and/or update Kernel images.") baseCmd.PersistentFlags().BoolVarP(&buildRuntimeOverlay, "runtime", "R", false, "Build and/or update runtime overlays") baseCmd.PersistentFlags().BoolVarP(&buildSystemOverlay, "system", "S", false, "Build and/or update system overlays") diff --git a/internal/app/wwctl/node/add/root.go b/internal/app/wwctl/node/add/root.go index f794ef80..99a4a75c 100644 --- a/internal/app/wwctl/node/add/root.go +++ b/internal/app/wwctl/node/add/root.go @@ -22,7 +22,7 @@ var ( func init() { baseCmd.PersistentFlags().StringVarP(&SetGroup, "group", "g", "default", "Group to add nodes to") baseCmd.PersistentFlags().StringVarP(&SetController, "controller", "c", "localhost", "Controller to add nodes to") - baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "n", "eth0", "Define the network device to configure") + baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "N", "eth0", "Define the network device to configure") baseCmd.PersistentFlags().StringVarP(&SetIpaddr, "ipaddr", "I", "", "Set the node's network device IP address") baseCmd.PersistentFlags().StringVarP(&SetNetmask, "netmask", "M", "", "Set the node's network device netmask") baseCmd.PersistentFlags().StringVarP(&SetGateway, "gateway", "G", "", "Set the node's network device gateway") diff --git a/internal/app/wwctl/node/list/main.go b/internal/app/wwctl/node/list/main.go index 5cc65f95..b2103686 100644 --- a/internal/app/wwctl/node/list/main.go +++ b/internal/app/wwctl/node/list/main.go @@ -50,7 +50,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "ClusterName", node.ClusterName.Source(), node.ClusterName.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "Profiles", "--", strings.Join(node.Profiles, ",")) - fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "Vnfs", node.Vnfs.Source(), node.Vnfs.Print()) + fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "ContainerName", node.ContainerName.Source(), node.ContainerName.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "KernelVersion", node.KernelVersion.Source(), node.KernelVersion.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "KernelArgs", node.KernelArgs.Source(), node.KernelArgs.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "RuntimeOverlay", node.RuntimeOverlay.Source(), node.RuntimeOverlay.Print()) @@ -99,11 +99,11 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } else if ShowLong == true { - fmt.Printf("%-22s %-26s %-35s %s\n", "NODE NAME", "KERNEL VERSION", "VNFS IMAGE", "OVERLAYS (S/R)") + fmt.Printf("%-22s %-26s %-35s %s\n", "NODE NAME", "KERNEL VERSION", "CONTAINER", "OVERLAYS (S/R)") fmt.Println(strings.Repeat("=", 120)) for _, node := range nodes { - fmt.Printf("%-22s %-26s %-35s %s\n", node.Id.Get(), node.KernelVersion.Print(), node.Vnfs.Print(), node.SystemOverlay.Print()+"/"+node.RuntimeOverlay.Print()) + fmt.Printf("%-22s %-26s %-35s %s\n", node.Id.Get(), node.KernelVersion.Print(), node.ContainerName.Print(), node.SystemOverlay.Print()+"/"+node.RuntimeOverlay.Print()) } } else { diff --git a/internal/app/wwctl/node/set/main.go b/internal/app/wwctl/node/set/main.go index 37b94e39..a6dba01b 100644 --- a/internal/app/wwctl/node/set/main.go +++ b/internal/app/wwctl/node/set/main.go @@ -2,9 +2,9 @@ package set import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -44,17 +44,17 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } - if SetVnfs != "" { - if vnfs.ValidSource(SetVnfs) == true { - imageFile := vnfs.ImageFile(SetVnfs) + if SetContainer != "" { + if container.ValidSource(SetContainer) == true { + imageFile := container.ImageFile(SetContainer) if util.IsFile(imageFile) == false { - wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + wwlog.Printf(wwlog.ERROR, "Container has not been built: %s\n", SetContainer) if SetForce == false { os.Exit(1) } } } else { - wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + wwlog.Printf(wwlog.ERROR, "Container does not exist: %s\n", SetContainer) if SetForce == false { os.Exit(1) } @@ -74,10 +74,10 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } } - if SetVnfs != "" { - wwlog.Printf(wwlog.VERBOSE, "Node: %s, Setting vnfs to: %s\n", n.Id.Get(), SetVnfs) + if SetContainer != "" { + wwlog.Printf(wwlog.VERBOSE, "Node: %s, Setting container name to: %s\n", n.Id.Get(), SetContainer) - n.Vnfs.Set(SetVnfs) + n.ContainerName.Set(SetContainer) err := nodeDB.NodeUpdate(n) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) diff --git a/internal/app/wwctl/node/set/root.go b/internal/app/wwctl/node/set/root.go index aa21b77c..24cb0f73 100644 --- a/internal/app/wwctl/node/set/root.go +++ b/internal/app/wwctl/node/set/root.go @@ -10,7 +10,7 @@ var ( RunE: CobraRunE, } SetComment string - SetVnfs string + SetContainer string SetKernel string SetNetDev string SetIpaddr string @@ -35,9 +35,9 @@ var ( func init() { baseCmd.PersistentFlags().StringVar(&SetComment, "comment", "", "Set a comment for this node") - baseCmd.PersistentFlags().StringVarP(&SetVnfs, "vnfs", "V", "", "Set node Virtual Node File System (VNFS)") + baseCmd.PersistentFlags().StringVarP(&SetContainer, "container", "C", "", "Set the container (VNFS) for this node") baseCmd.PersistentFlags().StringVarP(&SetKernel, "kernel", "K", "", "Set Kernel version for nodes") - baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "C", "", "Set the node's cluster name") + baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "c", "", "Set the node's cluster group") baseCmd.PersistentFlags().StringVarP(&SetIpxe, "ipxe", "P", "", "Set the node's iPXE template name") baseCmd.PersistentFlags().StringVarP(&SetRuntimeOverlay, "runtime", "R", "", "Set the node's runtime overlay") baseCmd.PersistentFlags().StringVarP(&SetSystemOverlay, "system", "S", "", "Set the node's system overlay") @@ -49,7 +49,7 @@ func init() { baseCmd.PersistentFlags().StringSliceVarP(&SetAddProfile, "addprofile", "p", []string{}, "Add Profile(s) to node") baseCmd.PersistentFlags().StringSliceVarP(&SetDelProfile, "delprofile", "r", []string{}, "Remove Profile(s) to node") - baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "n", "", "Define the network device to configure") + baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "N", "", "Define the network device to configure") baseCmd.PersistentFlags().StringVarP(&SetIpaddr, "ipaddr", "I", "", "Set the node's network device IP address") baseCmd.PersistentFlags().StringVarP(&SetNetmask, "netmask", "M", "", "Set the node's network device netmask") baseCmd.PersistentFlags().StringVarP(&SetGateway, "gateway", "G", "", "Set the node's network device gateway") diff --git a/internal/app/wwctl/profile/list/main.go b/internal/app/wwctl/profile/list/main.go index 20acec5a..1c4e8762 100644 --- a/internal/app/wwctl/profile/list/main.go +++ b/internal/app/wwctl/profile/list/main.go @@ -37,7 +37,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "Comment", profile.Comment.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "ClusterName", profile.ClusterName.Print()) - fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "Vnfs", profile.Vnfs.Print()) + fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "ContainerName", profile.ContainerName.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "KernelVersion", profile.KernelVersion.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "KernelArgs", profile.KernelArgs.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "RuntimeOverlay", profile.RuntimeOverlay.Print()) diff --git a/internal/app/wwctl/profile/set/main.go b/internal/app/wwctl/profile/set/main.go index f4058894..c146574b 100644 --- a/internal/app/wwctl/profile/set/main.go +++ b/internal/app/wwctl/profile/set/main.go @@ -2,9 +2,9 @@ package set import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -48,17 +48,17 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } - if SetVnfs != "" { - if vnfs.ValidSource(SetVnfs) == true { - imageFile := vnfs.ImageFile(SetVnfs) + if SetContainer != "" { + if container.ValidSource(SetContainer) == true { + imageFile := container.ImageFile(SetContainer) if util.IsFile(imageFile) == false { - wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + wwlog.Printf(wwlog.ERROR, "Container has not been built: %s\n", SetContainer) if SetForce == false { os.Exit(1) } } } else { - wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + wwlog.Printf(wwlog.ERROR, "Container name does not exist: %s\n", SetContainer) if SetForce == false { os.Exit(1) } @@ -88,10 +88,10 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } } - if SetVnfs != "" { - wwlog.Printf(wwlog.VERBOSE, "Profile: %s, Setting VNFS to: %s\n", p.Id, SetVnfs) + if SetContainer != "" { + wwlog.Printf(wwlog.VERBOSE, "Profile: %s, Setting Container name to: %s\n", p.Id, SetContainer) - p.Vnfs.Set(SetVnfs) + p.ContainerName.Set(SetContainer) err := nodeDB.ProfileUpdate(p) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) diff --git a/internal/app/wwctl/profile/set/root.go b/internal/app/wwctl/profile/set/root.go index d841534c..cd610b94 100644 --- a/internal/app/wwctl/profile/set/root.go +++ b/internal/app/wwctl/profile/set/root.go @@ -12,7 +12,7 @@ var ( SetAll bool SetForce bool SetComment string - SetVnfs string + SetContainer string SetKernel string SetClusterName string SetIpxe string @@ -31,9 +31,9 @@ var ( func init() { baseCmd.PersistentFlags().StringVar(&SetComment, "comment", "", "Set a comment for this node") - baseCmd.PersistentFlags().StringVarP(&SetVnfs, "vnfs", "V", "", "Set node Virtual Node File System (VNFS)") + baseCmd.PersistentFlags().StringVarP(&SetContainer, "container", "C", "", "Set the container (VNFS) for this node") baseCmd.PersistentFlags().StringVarP(&SetKernel, "kernel", "K", "", "Set Kernel version for nodes") - baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "C", "", "Set the node's cluster name") + baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "c", "", "Set the node's cluster group") baseCmd.PersistentFlags().StringVarP(&SetIpxe, "ipxe", "P", "", "Set the node's iPXE template name") baseCmd.PersistentFlags().StringVarP(&SetRuntimeOverlay, "runtime", "R", "", "Set the node's runtime overlay") baseCmd.PersistentFlags().StringVarP(&SetSystemOverlay, "system", "S", "", "Set the node's system overlay") @@ -41,7 +41,7 @@ func init() { baseCmd.PersistentFlags().StringVar(&SetIpmiUsername, "ipmiuser", "", "Set the node's IPMI username") baseCmd.PersistentFlags().StringVar(&SetIpmiPassword, "ipmipass", "", "Set the node's IPMI password") - baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "n", "", "Define the network device to configure") + baseCmd.PersistentFlags().StringVarP(&SetNetDev, "netdev", "N", "", "Define the network device to configure") baseCmd.PersistentFlags().StringVarP(&SetIpaddr, "ipaddr", "I", "", "Set the node's network device IP address") baseCmd.PersistentFlags().StringVarP(&SetNetmask, "netmask", "M", "", "Set the node's network device netmask") baseCmd.PersistentFlags().StringVarP(&SetGateway, "gateway", "G", "", "Set the node's network device gateway") diff --git a/internal/app/wwctl/ready/main.go b/internal/app/wwctl/ready/main.go index 1369b7fc..fec73de0 100644 --- a/internal/app/wwctl/ready/main.go +++ b/internal/app/wwctl/ready/main.go @@ -3,10 +3,10 @@ package ready import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" @@ -36,8 +36,8 @@ func CobraRunE(cmd *cobra.Command, args []string) error { var runtimeo_good bool status := true - if node.Vnfs.Get() != "" { - vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) + if node.ContainerName.Get() != "" { + vnfsImage := container.ImageFile(node.ContainerName.Get()) if util.IsFile(vnfsImage) == true { vnfs_good = true diff --git a/internal/app/wwctl/root.go b/internal/app/wwctl/root.go index 03efec60..72119651 100644 --- a/internal/app/wwctl/root.go +++ b/internal/app/wwctl/root.go @@ -1,13 +1,13 @@ package wwctl import ( + "github.com/hpcng/warewulf/internal/app/wwctl/container" "github.com/hpcng/warewulf/internal/app/wwctl/kernel" "github.com/hpcng/warewulf/internal/app/wwctl/node" "github.com/hpcng/warewulf/internal/app/wwctl/overlay" "github.com/hpcng/warewulf/internal/app/wwctl/profile" "github.com/hpcng/warewulf/internal/app/wwctl/ready" "github.com/hpcng/warewulf/internal/app/wwctl/service" - "github.com/hpcng/warewulf/internal/app/wwctl/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" @@ -32,7 +32,7 @@ func init() { rootCmd.AddCommand(overlay.GetCommand()) // rootCmd.AddCommand(controller.GetCommand()) - rootCmd.AddCommand(vnfs.GetCommand()) + rootCmd.AddCommand(container.GetCommand()) rootCmd.AddCommand(node.GetCommand()) rootCmd.AddCommand(kernel.GetCommand()) // rootCmd.AddCommand(group.GetCommand()) diff --git a/internal/app/wwctl/vnfs/build/main.go b/internal/app/wwctl/vnfs/build/main.go deleted file mode 100644 index fa2330e1..00000000 --- a/internal/app/wwctl/vnfs/build/main.go +++ /dev/null @@ -1,79 +0,0 @@ -package build - -import ( - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "github.com/hpcng/warewulf/internal/pkg/wwlog" - "github.com/spf13/cobra" - "os" -) - -func CobraRunE(cmd *cobra.Command, args []string) error { - - for _, arg := range args { - if vnfs.ValidSource(arg) == false { - wwlog.Printf(wwlog.ERROR, "VNFS name does not exist: %s\n", arg) - os.Exit(1) - } - - vnfs.Build(arg, BuildForce) - } - - /* - var nodes []node.NodeInfo - set := make(map[string]int) - - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - if len(args) == 1 && ByNode == true { - var err error - nodes, err = n.SearchByName(args[0]) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) - os.Exit(1) - } - - for _, node := range nodes { - if node.Vnfs.Defined() == true { - set[node.Vnfs.Get()]++ - } - } - - } else if BuildAll == true { - var err error - nodes, err = n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) - os.Exit(1) - } - - for _, node := range nodes { - if node.Vnfs.Defined() == true { - wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) - set[node.Vnfs.Get()]++ - } - } - - } else if len(args) == 1 { - set[args[0]]++ - } else { - cmd.Usage() - os.Exit(1) - } - - for v := range set { - fmt.Printf("Building VNFS: %s\n", v) - err := vnfs.Build(v, BuildForce) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } - - */ - - return nil -} diff --git a/internal/app/wwctl/vnfs/build/root.go b/internal/app/wwctl/vnfs/build/root.go deleted file mode 100644 index 50dbdfdc..00000000 --- a/internal/app/wwctl/vnfs/build/root.go +++ /dev/null @@ -1,28 +0,0 @@ -package build - -import "github.com/spf13/cobra" - -var ( - baseCmd = &cobra.Command{ - Use: "build (vnfs location | node search pattern)", - Short: "VNFS Image Build", - Long: "VNFS kernel images", - RunE: CobraRunE, - Args: cobra.RangeArgs(0,1), - - } - BuildForce bool - BuildAll bool - ByNode bool -) - -func init() { - baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "(re)Build all VNFS images for all nodes") - baseCmd.PersistentFlags().BoolVarP(&BuildForce, "force", "f", false, "Force rebuild, even if it isn't necessary") - baseCmd.PersistentFlags().BoolVarP(&ByNode, "node", "n", false, "Build VNFS for a particular node(s)") -} - -// GetRootCommand returns the root cobra.Command for the application. -func GetCommand() *cobra.Command { - return baseCmd -} \ No newline at end of file diff --git a/internal/app/wwctl/vnfs/list/main.go b/internal/app/wwctl/vnfs/list/main.go deleted file mode 100644 index 2499fbac..00000000 --- a/internal/app/wwctl/vnfs/list/main.go +++ /dev/null @@ -1,89 +0,0 @@ -package list - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "github.com/hpcng/warewulf/internal/pkg/wwlog" - "github.com/spf13/cobra" - "os" -) - -func CobraRunE(cmd *cobra.Command, args []string) error { - - sources, err := vnfs.ListSources() - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - - nconfig, _ := node.New() - nodes, _ := nconfig.FindAllNodes() - nodemap := make(map[string]int) - - for _, n := range nodes { - nodemap[n.Vnfs.Get()]++ - } - - fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") - for _, source := range sources { - image := vnfs.ImageFile(source) - - if nodemap[source] == 0 { - nodemap[source] = 0 - } - fmt.Printf("%-35s %-6t %-6d\n", source, util.IsFile(image), nodemap[source]) - - } - - /* - nconfig, _ := node.New() - nodes, _ := nconfig.FindAllNodes() - nodemap := make(map[string]int) - - for _, n := range nodes { - nodemap[n.Vnfs.Get()]++ - } - - images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) - - fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") - fmt.Println(strings.Repeat("=", 80)) - - for _, file := range images { - v, err := vnfs.Load(file.Name()) - if err == nil { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[v.Source] > 0 { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) - } else { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) - } - - } - continue - - if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[file.Name()] > 0 { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) - } else { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) - } - } - } - */ - - return nil -} diff --git a/internal/app/wwctl/vnfs/list/root.go b/internal/app/wwctl/vnfs/list/root.go deleted file mode 100644 index 80216bb9..00000000 --- a/internal/app/wwctl/vnfs/list/root.go +++ /dev/null @@ -1,25 +0,0 @@ -package list - -import "github.com/spf13/cobra" - -var ( - baseCmd = &cobra.Command{ - Use: "list", - Short: "List VNFS images", - Long: "List VNFS images ", - RunE: CobraRunE, - } - SystemOverlay bool - BuildAll bool -) - -func init() { - //baseCmd.PersistentFlags().BoolVarP(&SystemOverlay, "system", "s", false, "Show System Overlays as well") - //baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "Build all overlays (runtime and system)") - -} - -// GetRootCommand returns the root cobra.Command for the application. -func GetCommand() *cobra.Command { - return baseCmd -} diff --git a/internal/app/wwctl/vnfs/pull/main.go b/internal/app/wwctl/vnfs/pull/main.go deleted file mode 100644 index b85466fe..00000000 --- a/internal/app/wwctl/vnfs/pull/main.go +++ /dev/null @@ -1,54 +0,0 @@ -package pull - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "github.com/hpcng/warewulf/internal/pkg/wwlog" - "github.com/spf13/cobra" - "os" - "path" -) - -func CobraRunE(cmd *cobra.Command, args []string) error { - var name string - uri := args[0] - - if len(args) == 2 { - name = args[1] - } else { - name = path.Base(uri) - fmt.Printf("Setting VNFS name: %s\n", name) - } - - if vnfs.ValidName(name) == false { - wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", name) - os.Exit(1) - } - - fullPath := vnfs.SourceDir(name) - - if util.IsDir(fullPath) == true { - if SetForce == true { - wwlog.Printf(wwlog.WARN, "Overwriting existing VNFS\n") - err := os.RemoveAll(fullPath) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } else if SetUpdate == true { - wwlog.Printf(wwlog.WARN, "Updating existing VNFS\n") - } else { - wwlog.Printf(wwlog.ERROR, "VNFS Name exists, specify --force, --update, or choose a different name: %s\n", name) - os.Exit(1) - } - } - - err := vnfs.PullURI(uri, name) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not pull image: %s\n", err) - os.Exit(1) - } - - return nil -} diff --git a/internal/app/wwctl/vnfs/pull/root.go b/internal/app/wwctl/vnfs/pull/root.go deleted file mode 100644 index 93d88329..00000000 --- a/internal/app/wwctl/vnfs/pull/root.go +++ /dev/null @@ -1,26 +0,0 @@ -package pull - -import "github.com/spf13/cobra" - -var ( - baseCmd = &cobra.Command{ - Use: "pull", - Short: "Pull Source OCI VNFS images", - Long: "Pull Source OCI VNFS images ", - RunE: CobraRunE, - Args: cobra.MinimumNArgs(1), - } - SetForce bool - SetUpdate bool -) - -func init() { - baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force overwrite of an existing VNFS") - baseCmd.PersistentFlags().BoolVarP(&SetUpdate, "update", "u", false, "Update and overwrite an existing VNFS") - -} - -// GetRootCommand returns the root cobra.Command for the application. -func GetCommand() *cobra.Command { - return baseCmd -} diff --git a/internal/app/wwctl/vnfs/root.go b/internal/app/wwctl/vnfs/root.go deleted file mode 100644 index 0b10b5c2..00000000 --- a/internal/app/wwctl/vnfs/root.go +++ /dev/null @@ -1,29 +0,0 @@ -package vnfs - -import ( - "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/build" - "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/list" - "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/pull" - "github.com/spf13/cobra" -) - -var ( - baseCmd = &cobra.Command{ - Use: "vnfs", - Short: "VNFS image management", - Long: "Virtual Node File System (VNFS) image management", - } - test bool -) - -func init() { - baseCmd.AddCommand(build.GetCommand()) - baseCmd.AddCommand(list.GetCommand()) - baseCmd.AddCommand(pull.GetCommand()) - -} - -// GetRootCommand returns the root cobra.Command for the application. -func GetCommand() *cobra.Command { - return baseCmd -} diff --git a/internal/pkg/vnfs/build.go b/internal/pkg/container/build.go similarity index 98% rename from internal/pkg/vnfs/build.go rename to internal/pkg/container/build.go index 101d6b7c..80ce972e 100644 --- a/internal/pkg/vnfs/build.go +++ b/internal/pkg/container/build.go @@ -1,4 +1,4 @@ -package vnfs +package container import ( "fmt" diff --git a/internal/pkg/vnfs/pull.go b/internal/pkg/container/pull.go similarity index 97% rename from internal/pkg/vnfs/pull.go rename to internal/pkg/container/pull.go index f9fd75aa..b559c4fe 100644 --- a/internal/pkg/vnfs/pull.go +++ b/internal/pkg/container/pull.go @@ -1,4 +1,4 @@ -package vnfs +package container import ( "context" diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/container/vnfs.go similarity index 95% rename from internal/pkg/vnfs/vnfs.go rename to internal/pkg/container/vnfs.go index ac40fc9e..0a3e6991 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/container/vnfs.go @@ -1,4 +1,4 @@ -package vnfs +package container import ( "github.com/hpcng/warewulf/internal/pkg/config" @@ -31,7 +31,7 @@ func RootFsDir(name string) string { } func ImageParentDir() string { - return path.Join(config.LocalStateDir, "provision/vnfs/") + return path.Join(config.LocalStateDir, "provision/container/") } func ImageFile(name string) string { diff --git a/internal/pkg/node/constructors.go b/internal/pkg/node/constructors.go index 70070760..c7baedaa 100644 --- a/internal/pkg/node/constructors.go +++ b/internal/pkg/node/constructors.go @@ -59,7 +59,7 @@ func (self *nodeYaml) FindAllNodes() ([]NodeInfo, error) { n.Id.Set(nodename) n.Comment.Set(node.Comment) - n.Vnfs.Set(node.Vnfs) + n.ContainerName.Set(node.ContainerName) n.KernelVersion.Set(node.KernelVersion) n.KernelArgs.Set(node.KernelArgs) n.ClusterName.Set(node.ClusterName) @@ -97,7 +97,7 @@ func (self *nodeYaml) FindAllNodes() ([]NodeInfo, error) { n.Comment.SetAlt(self.NodeProfiles[p].Comment, pstring) n.ClusterName.SetAlt(self.NodeProfiles[p].ClusterName, pstring) - n.Vnfs.SetAlt(self.NodeProfiles[p].Vnfs, pstring) + n.ContainerName.SetAlt(self.NodeProfiles[p].ContainerName, pstring) n.KernelVersion.SetAlt(self.NodeProfiles[p].KernelVersion, pstring) n.KernelArgs.SetAlt(self.NodeProfiles[p].KernelArgs, pstring) n.Ipxe.SetAlt(self.NodeProfiles[p].Ipxe, pstring) @@ -140,7 +140,7 @@ func (self *nodeYaml) FindAllProfiles() ([]NodeInfo, error) { p.Id.Set(name) p.Comment.Set(profile.Comment) - p.Vnfs.Set(profile.Vnfs) + p.ContainerName.Set(profile.ContainerName) p.Ipxe.Set(profile.Ipxe) p.KernelVersion.Set(profile.KernelVersion) p.KernelArgs.Set(profile.KernelArgs) diff --git a/internal/pkg/node/datastructure.go b/internal/pkg/node/datastructure.go index 8cf4219a..cd7ea2eb 100644 --- a/internal/pkg/node/datastructure.go +++ b/internal/pkg/node/datastructure.go @@ -19,7 +19,7 @@ type NodeConf struct { Comment string `yaml:"comment,omitempty"` Disabled bool `yaml:"disabled,omitempty"` ClusterName string `yaml:"cluster name,omitempty"` - Vnfs string `yaml:"vnfs,omitempty"` + ContainerName string `yaml:"container name,omitempty"` Ipxe string `yaml:"ipxe template,omitempty"` KernelVersion string `yaml:"kernel version,omitempty"` KernelArgs string `yaml:"kernel args,omitempty"` @@ -60,7 +60,7 @@ type NodeInfo struct { Cid Entry Comment Entry ClusterName Entry - Vnfs Entry + ContainerName Entry Ipxe Entry KernelVersion Entry KernelArgs Entry diff --git a/internal/pkg/node/modifiers.go b/internal/pkg/node/modifiers.go index 49bf6f06..9b5773f4 100644 --- a/internal/pkg/node/modifiers.go +++ b/internal/pkg/node/modifiers.go @@ -54,7 +54,7 @@ func (self *nodeYaml) NodeUpdate(node NodeInfo) error { } self.Nodes[nodeID].Comment = node.Comment.GetReal() - self.Nodes[nodeID].Vnfs = node.Vnfs.GetReal() + self.Nodes[nodeID].ContainerName = node.ContainerName.GetReal() self.Nodes[nodeID].ClusterName = node.ClusterName.GetReal() self.Nodes[nodeID].Ipxe = node.Ipxe.GetReal() self.Nodes[nodeID].KernelVersion = node.KernelVersion.GetReal() @@ -125,7 +125,7 @@ func (self *nodeYaml) ProfileUpdate(profile NodeInfo) error { return errors.New("Profile name does not exist: " + profileID) } self.NodeProfiles[profileID].Comment = profile.Comment.GetReal() - self.NodeProfiles[profileID].Vnfs = profile.Vnfs.GetReal() + self.NodeProfiles[profileID].ContainerName = profile.ContainerName.GetReal() self.NodeProfiles[profileID].Ipxe = profile.Ipxe.GetReal() self.NodeProfiles[profileID].ClusterName = profile.ClusterName.GetReal() self.NodeProfiles[profileID].KernelVersion = profile.KernelVersion.GetReal() diff --git a/internal/pkg/oci/cache.go b/internal/pkg/oci/cache.go index 369efaf4..1f4231b4 100644 --- a/internal/pkg/oci/cache.go +++ b/internal/pkg/oci/cache.go @@ -10,7 +10,7 @@ import ( ) const ( - defaultCachePath = "/var/warewulf/vnfs-cache/oci/" + defaultCachePath = "/var/warewulf/container-cache/oci/" blobPrefix = "blobs" rootfsPrefix = "rootfs" ) diff --git a/internal/pkg/overlay/funcmap.go b/internal/pkg/overlay/funcmap.go index 7843a4ae..fbc08f85 100644 --- a/internal/pkg/overlay/funcmap.go +++ b/internal/pkg/overlay/funcmap.go @@ -1,7 +1,7 @@ package overlay import ( - "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/container" "github.com/hpcng/warewulf/internal/pkg/wwlog" "io/ioutil" "path" @@ -17,24 +17,24 @@ func templateFileInclude(path string) string { return strings.TrimSuffix(string(content), "\n") } -func templateVnfsFileInclude(vnfsname string, filepath string) string { - wwlog.Printf(wwlog.DEBUG, "Including VNFS file into template: %s: %s\n", vnfsname, filepath) +func templateContainerFileInclude(containername string, filepath string) string { + wwlog.Printf(wwlog.DEBUG, "Including VNFS file into template: %s: %s\n", containername, filepath) - if vnfsname == "" { - wwlog.Printf(wwlog.WARN, "VNFS not set for template import request: %s: %s\n", vnfsname, filepath) + if containername == "" { + wwlog.Printf(wwlog.WARN, "VNFS not set for template import request: %s: %s\n", containername, filepath) return "" } - if vnfs.ValidSource(vnfsname) == false { - wwlog.Printf(wwlog.WARN, "Template required VNFS does not exist: %s\n", vnfsname) + if container.ValidSource(containername) == false { + wwlog.Printf(wwlog.WARN, "Template required VNFS does not exist: %s\n", containername) return "" } - vnfsDir := vnfs.RootFsDir(vnfsname) + containerDir := container.RootFsDir(containername) - wwlog.Printf(wwlog.DEBUG, "Including file from VNFS: %s:%s\n", vnfsDir, filepath) + wwlog.Printf(wwlog.DEBUG, "Including file from container: %s:%s\n", containerDir, filepath) - content, err := ioutil.ReadFile(path.Join(vnfsDir, filepath)) + content, err := ioutil.ReadFile(path.Join(containerDir, filepath)) if err != nil { wwlog.Printf(wwlog.ERROR, "Template include: %s\n", err) diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 9f1e938b..3ac359a4 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -18,18 +18,16 @@ import ( ) type TemplateStruct struct { - Self struct { - Id string - Hostname string - GroupName string - Vnfs string - IpmiIpaddr string - IpmiNetmask string - IpmiUserName string - IpmiPassword string - NetDevs map[string]*node.NetDevs - } - AllNodes []node.NodeInfo + Id string + Hostname string + GroupName string + Container string + IpmiIpaddr string + IpmiNetmask string + IpmiUserName string + IpmiPassword string + NetDevs map[string]*node.NetDevs + AllNodes []node.NodeInfo } func BuildSystemOverlay(nodeList []node.NodeInfo) error { @@ -131,22 +129,22 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { wwlog.Printf(wwlog.DEBUG, "Processing overlay for node: %s\n", n.Id.Get()) - t.Self.Id = n.Id.Get() - t.Self.Hostname = n.Id.Get() - t.Self.Vnfs = n.Vnfs.Get() - t.Self.IpmiIpaddr = n.IpmiIpaddr.Get() - t.Self.IpmiNetmask = n.IpmiNetmask.Get() - t.Self.IpmiUserName = n.IpmiUserName.Get() - t.Self.IpmiPassword = n.IpmiPassword.Get() - t.Self.NetDevs = make(map[string]*node.NetDevs) + t.Id = n.Id.Get() + t.Hostname = n.Id.Get() + t.Container = n.ContainerName.Get() + t.IpmiIpaddr = n.IpmiIpaddr.Get() + t.IpmiNetmask = n.IpmiNetmask.Get() + t.IpmiUserName = n.IpmiUserName.Get() + t.IpmiPassword = n.IpmiPassword.Get() + t.NetDevs = make(map[string]*node.NetDevs) for devname, netdev := range n.NetDevs { var nd node.NetDevs - t.Self.NetDevs[devname] = &nd - t.Self.NetDevs[devname].Hwaddr = netdev.Hwaddr.Get() - t.Self.NetDevs[devname].Ipaddr = netdev.Ipaddr.Get() - t.Self.NetDevs[devname].Netmask = netdev.Netmask.Get() - t.Self.NetDevs[devname].Gateway = netdev.Gateway.Get() - t.Self.NetDevs[devname].Type = netdev.Type.Get() + t.NetDevs[devname] = &nd + t.NetDevs[devname].Hwaddr = netdev.Hwaddr.Get() + t.NetDevs[devname].Ipaddr = netdev.Ipaddr.Get() + t.NetDevs[devname].Netmask = netdev.Netmask.Get() + t.NetDevs[devname].Gateway = netdev.Gateway.Get() + t.NetDevs[devname].Type = netdev.Type.Get() } t.AllNodes = allNodes @@ -203,8 +201,8 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { destFile := strings.TrimSuffix(location, ".ww") tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{ - "Include": templateFileInclude, - "IncludeFromVnfs": templateVnfsFileInclude, + "Include": templateFileInclude, + "IncludeFrom": templateContainerFileInclude, }).ParseGlob(path.Join(OverlayDir, destFile+".ww*")) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) diff --git a/internal/pkg/vnfs_old/docker.go b/internal/pkg/vnfs_old/docker.go index a060c406..46bf8111 100644 --- a/internal/pkg/vnfs_old/docker.go +++ b/internal/pkg/vnfs_old/docker.go @@ -14,7 +14,7 @@ func BuildDocker(vnfs VnfsObject, buildForce bool) { wwlog.Printf(wwlog.VERBOSE, "Building OCI Container: %s\n", vnfs.Source) OciCacheDir := config.LocalStateDir + "/oci" - VnfsHashDir := config.LocalStateDir + "/oci/vnfs" + VnfsHashDir := config.LocalStateDir + "/oci/container" c, err := oci.NewCache(oci.OptSetCachePath(OciCacheDir)) if err != nil { diff --git a/internal/pkg/vnfs_old/vnfs.go b/internal/pkg/vnfs_old/vnfs.go index 32454fb9..79ae7a02 100644 --- a/internal/pkg/vnfs_old/vnfs.go +++ b/internal/pkg/vnfs_old/vnfs.go @@ -24,8 +24,8 @@ func Load(name string) (VnfsObject, error) { var ret VnfsObject if name == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") + wwlog.Printf(wwlog.DEBUG, "Called container.Load() without a name, returning error\n") + return ret, errors.New("Called container.Load() without a VNFS name") } pathFriendlyName := CleanName(name) @@ -69,8 +69,8 @@ func New(source string) (VnfsObject, error) { var ret VnfsObject if source == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") + wwlog.Printf(wwlog.DEBUG, "Called container.Load() without a name, returning error\n") + return ret, errors.New("Called container.Load() without a VNFS name") } if util.IsFile(ret.Config) { diff --git a/overlays/runtime/default/etc/group.ww b/overlays/runtime/default/etc/group.ww index ca59dd40..8005528f 100644 --- a/overlays/runtime/default/etc/group.ww +++ b/overlays/runtime/default/etc/group.ww @@ -1,2 +1,2 @@ -{{IncludeFromVnfs $.Self.Vnfs "/etc/group"}} +{{IncludeFrom $.Container "/etc/group"}} {{Include "/etc/group"}} diff --git a/overlays/runtime/default/etc/passwd.ww b/overlays/runtime/default/etc/passwd.ww index bd653b8a..e436e34b 100644 --- a/overlays/runtime/default/etc/passwd.ww +++ b/overlays/runtime/default/etc/passwd.ww @@ -1,3 +1,3 @@ root::0:0:root:/root:/bin/bash -{{IncludeFromVnfs $.Self.Vnfs "/etc/passwd"}} +{{IncludeFrom $.Container "/etc/passwd"}} {{Include "/etc/passwd"}} diff --git a/overlays/system/default/etc/hostname.ww b/overlays/system/default/etc/hostname.ww index 5bbffc40..b195826a 100644 --- a/overlays/system/default/etc/hostname.ww +++ b/overlays/system/default/etc/hostname.ww @@ -1 +1 @@ -{{$.Self.Id}} +{{$.Id}} diff --git a/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww b/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww index ce5ae245..a2fbee93 100644 --- a/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww +++ b/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww @@ -1,8 +1,8 @@ DEVICE=eth0 NAME=eth0 -IPADDR={{$.Self.NetDevs.eth0.Ipaddr}} -NETMASK={{$.Self.NetDevs.eth0.Netmask}} -GATEWAY={{$.Self.NetDevs.eth0.Gateway}} -HWADDR={{$.Self.NetDevs.eth0.Hwaddr}} +IPADDR={{$.NetDevs.eth0.Ipaddr}} +NETMASK={{$.NetDevs.eth0.Netmask}} +GATEWAY={{$.NetDevs.eth0.Gateway}} +HWADDR={{$.NetDevs.eth0.Hwaddr}} BOOTPROTO=static ONBOOT=yes diff --git a/overlays/system/default/etc/sysconfig/network.ww b/overlays/system/default/etc/sysconfig/network.ww index bcbd324d..08ba4073 100644 --- a/overlays/system/default/etc/sysconfig/network.ww +++ b/overlays/system/default/etc/sysconfig/network.ww @@ -1,2 +1,2 @@ NETWORKING=yes -HOSTNAME={{$.Self.Id}} +HOSTNAME={{$.Id}} diff --git a/overlays/system/default/ipmi.ww b/overlays/system/default/ipmi.ww index afd051bb..672ab940 100755 --- a/overlays/system/default/ipmi.ww +++ b/overlays/system/default/ipmi.ww @@ -8,10 +8,10 @@ if [ ! -e /dev/ipmi0 ]; then ls -la /dev/ipmi0 fi -IP="{{$.Self.IpmiIpaddr}}" -NETMASK="{{$.Self.IpmiNetmask}}" -USER="{{$.Self.IpmiUserName}}" -PASS="{{$.Self.IpmiPassword}}" +IP="{{$.IpmiIpaddr}}" +NETMASK="{{$.IpmiNetmask}}" +USER="{{$.IpmiUserName}}" +PASS="{{$.IpmiPassword}}" echo IP is $IP echo NETMASK is $NETMASK From f057db35727c8913e9c6d14d8c5e36073c232faa Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 01:05:37 -0800 Subject: [PATCH 3/9] Updated README --- README.md | 74 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 68905715..bf73582b 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ To install these and compile Warewulf, do the following: ``` sudo yum install epel-release -sudo yum install golang tftp-server dhcp +sudo yum install --tolerant golang tftp-server dhcp dhcp-server sudo systemctl stop firewalld sudo systemctl disable firewalld @@ -51,59 +51,71 @@ netmask: 255.255.255.0 warewulf: port: 9873 secure: true + update interval: 60 dhcp: enabled: true range start: 192.168.1.150 range end: 192.168.1.200 template: default + systemd name: dhcpd +tftp: + enabled: true + tftproot: /var/lib/tftpboot + systemd name: tftp ``` +Note: You may need to change the systemd service names for your distribution. + Once it has been configured, you can have Warewulf configure the services: ``` -sudo ./wwctl service dhcp -c -# sudo ./wwctl service tftp -c +sudo ./wwctl service -a ``` - -#### Use the controller's kernel and default VNFS into your "default" node profile: -Next we are going to start configuring Warewulf. Use the following two commands to -set the kernel and VNFS for the "default" profile which all nodes will utilize, and -then configure node specific information as you add a new node to the configuration: - -``` -sudo ./wwctl profile set default -K $(uname -r) -V docker://warewulf/centos-8 -sudo ./wwctl node add n0000.cluster --netdev eth0 -I 192.168.1.100 -M 255.255.255.0 -G 192.168.1.1 -H 00:0c:29:23:8b:48 -``` - - -#### Build the kernel, VNFS, and overlays: +#### Pull and build the VNFS container and kernel: Once you have added the node, you can start building the needed bootable components. There are three major groups of data to provision: 1. Kernel: This is the boot kernel and driver overlay pair. The `wwctl kernel build` -command will create these files. The `-a` option will scan your configuration looking -for all needed kernel images and then go through and build them all. The caveat is -that all these kernels must be installed to your host controller node. + command will create these files. The `-a` option will scan your configuration looking + for all needed kernel images and then go through and build them all. The caveat is + that all these kernels must be installed to your host controller node. 1. VNFS: The "VNFS" is the "Virtual Node File System", and that is the template that -nodes will be provisioned to boot into. Warewulf v4 can support standard "chroot" -style VNFS formats (e.g. same as Warewulf 3 and/or Singularity Sandboxes) as well as -OCI (Open Container Initiative) formats which include Docker containers and containers -in Docker Hub. As you can see in the above `wwctl profile set` command, we configured -the VNFS to be a container hosted in Docker Hub. This can also be a local path or a -container in a `docker-daemon`. + nodes will be provisioned to boot into. Warewulf v4 can support standard "chroot" + style VNFS formats (e.g. same as Warewulf 3 and/or Singularity Sandboxes) as well as + OCI (Open Container Initiative) formats which include Docker containers and containers + in Docker Hub. As you can see in the above `wwctl profile set` command, we configured + the VNFS to be a container hosted in Docker Hub. This can also be a local path or a + container in a `docker-daemon`. 1. Overlays: There are two types of overlays, "system" and "runtime". The difference is -that the system overlay is provisioned before `/sbin/init` is called and the runtime -overlay is provisioned after `/sbin/init` is called and is done from the booted operating -system at periodic intervals (the time of this writing, it is ever 30 seconds). + that the system overlay is provisioned before `/sbin/init` is called and the runtime + overlay is provisioned after `/sbin/init` is called and is done from the booted operating + system at periodic intervals (the time of this writing, it is ever 30 seconds). + ``` -sudo ./wwctl kernel build -a -sudo ./wwctl vnfs build -a -sudo ./wwctl overlay build -sa +sudo ./wwctl container pull docker://warewulf/centos-7 centos-7 +sudo ./wwctl container build centos-7 +sudo ./wwctl kernel build $(uname -r) +``` + +#### Set up the default node profile + +``` +sudo ./wwctl profile set default -K $(uname -r) -C centos-7 +sudo ./wwctl profile set default --netdev eth0 -M 255.255.255.0 -G 192.168.1.1 +sudo ./wwctl profile list +``` + +#### Add a node and build node specific overlays + +``` +sudo ./wwctl node add n0000.cluster --netdev eth0 -I 192.168.1.100 -H 00:0c:29:23:8b:48 +sudo ./wwctl node list -a n0000 +sudo ./wwctl overlay build -a ``` #### Start the Warewulf daemon: From 6c1c9d17692690aa9ff847a90a0b28ae4b4e57ab Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 01:11:57 -0800 Subject: [PATCH 4/9] Not sure why my .gitignore is causing some files to be skipped... Have to look into that. --- README.md | 2 +- internal/app/warewulfd/response/container.go | 34 ++++++++ internal/app/wwctl/container/build/main.go | 86 +++++++++++++++++++ internal/app/wwctl/container/build/root.go | 25 ++++++ internal/app/wwctl/container/list/main.go | 89 ++++++++++++++++++++ internal/app/wwctl/container/list/root.go | 25 ++++++ internal/app/wwctl/container/pull/main.go | 59 +++++++++++++ internal/app/wwctl/container/pull/root.go | 28 ++++++ internal/app/wwctl/container/root.go | 29 +++++++ 9 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 internal/app/warewulfd/response/container.go create mode 100644 internal/app/wwctl/container/build/main.go create mode 100644 internal/app/wwctl/container/build/root.go create mode 100644 internal/app/wwctl/container/list/main.go create mode 100644 internal/app/wwctl/container/list/root.go create mode 100644 internal/app/wwctl/container/pull/main.go create mode 100644 internal/app/wwctl/container/pull/root.go create mode 100644 internal/app/wwctl/container/root.go diff --git a/README.md b/README.md index bf73582b..47b51b27 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ There are three major groups of data to provision: for all needed kernel images and then go through and build them all. The caveat is that all these kernels must be installed to your host controller node. -1. VNFS: The "VNFS" is the "Virtual Node File System", and that is the template that +1. Container/VNFS: The "VNFS" is the "Virtual Node File System", and that is the template that nodes will be provisioned to boot into. Warewulf v4 can support standard "chroot" style VNFS formats (e.g. same as Warewulf 3 and/or Singularity Sandboxes) as well as OCI (Open Container Initiative) formats which include Docker containers and containers diff --git a/internal/app/warewulfd/response/container.go b/internal/app/warewulfd/response/container.go new file mode 100644 index 00000000..da58d804 --- /dev/null +++ b/internal/app/warewulfd/response/container.go @@ -0,0 +1,34 @@ +package response + +import ( + "github.com/hpcng/warewulf/internal/pkg/container" + "log" + "net/http" +) + +func ContainerSend(w http.ResponseWriter, req *http.Request) { + + node, err := getSanity(req) + if err != nil { + w.WriteHeader(404) + log.Panicln(err) + return + } + + if node.ContainerName.Defined() == true { + containerImage := container.ImageFile(node.ContainerName.Get()) + + err = sendFile(w, containerImage, node.Id.Get()) + if err != nil { + log.Printf("ERROR1: %s\n", err) + w.WriteHeader(503) + } else { + log.Printf("SEND: %15s: %s\n", node.Id.Get(), containerImage) + } + } else { + w.WriteHeader(503) + log.Printf("ERROR: No Container set for node %s\n", node.Id.Get()) + } + + return +} diff --git a/internal/app/wwctl/container/build/main.go b/internal/app/wwctl/container/build/main.go new file mode 100644 index 00000000..81b6bb66 --- /dev/null +++ b/internal/app/wwctl/container/build/main.go @@ -0,0 +1,86 @@ +package build + +import ( + "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var containers []string + + if BuildAll == true { + containers, _ = container.ListSources() + } else { + containers = args + } + + for _, c := range containers { + if container.ValidSource(c) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name does not exist: %s\n", c) + os.Exit(1) + } + + container.Build(c, BuildForce) + } + + /* + var nodes []node.NodeInfo + set := make(map[string]int) + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) == 1 && ByNode == true { + var err error + nodes, err = n.SearchByName(args[0]) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + set[node.Vnfs.Get()]++ + } + } + + } else if BuildAll == true { + var err error + nodes, err = n.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) + set[node.Vnfs.Get()]++ + } + } + + } else if len(args) == 1 { + set[args[0]]++ + } else { + cmd.Usage() + os.Exit(1) + } + + for v := range set { + fmt.Printf("Building VNFS: %s\n", v) + err := container.Build(v, BuildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } + + */ + + return nil +} diff --git a/internal/app/wwctl/container/build/root.go b/internal/app/wwctl/container/build/root.go new file mode 100644 index 00000000..f174053d --- /dev/null +++ b/internal/app/wwctl/container/build/root.go @@ -0,0 +1,25 @@ +package build + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "build (container location | node search pattern)", + Short: "VNFS Image Build", + Long: "VNFS kernel images", + RunE: CobraRunE, + Args: cobra.RangeArgs(0, 1), + } + BuildForce bool + BuildAll bool +) + +func init() { + baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "(re)Build all VNFS images for all nodes") + baseCmd.PersistentFlags().BoolVarP(&BuildForce, "force", "f", false, "Force rebuild, even if it isn't necessary") +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/container/list/main.go b/internal/app/wwctl/container/list/main.go new file mode 100644 index 00000000..132b118f --- /dev/null +++ b/internal/app/wwctl/container/list/main.go @@ -0,0 +1,89 @@ +package list + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + + sources, err := container.ListSources() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.ContainerName.Get()]++ + } + + fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") + for _, source := range sources { + image := container.ImageFile(source) + + if nodemap[source] == 0 { + nodemap[source] = 0 + } + fmt.Printf("%-35s %-6t %-6d\n", source, util.IsFile(image), nodemap[source]) + + } + + /* + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.Vnfs.Get()]++ + } + + images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) + + fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") + fmt.Println(strings.Repeat("=", 80)) + + for _, file := range images { + v, err := container.Load(file.Name()) + if err == nil { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[v.Source] > 0 { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) + } else { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) + } + + } + continue + + if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[file.Name()] > 0 { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) + } else { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) + } + } + } + */ + + return nil +} diff --git a/internal/app/wwctl/container/list/root.go b/internal/app/wwctl/container/list/root.go new file mode 100644 index 00000000..80216bb9 --- /dev/null +++ b/internal/app/wwctl/container/list/root.go @@ -0,0 +1,25 @@ +package list + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "list", + Short: "List VNFS images", + Long: "List VNFS images ", + RunE: CobraRunE, + } + SystemOverlay bool + BuildAll bool +) + +func init() { + //baseCmd.PersistentFlags().BoolVarP(&SystemOverlay, "system", "s", false, "Show System Overlays as well") + //baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "Build all overlays (runtime and system)") + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/container/pull/main.go b/internal/app/wwctl/container/pull/main.go new file mode 100644 index 00000000..f9a9919d --- /dev/null +++ b/internal/app/wwctl/container/pull/main.go @@ -0,0 +1,59 @@ +package pull + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" + "path" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var name string + uri := args[0] + + if len(args) == 2 { + name = args[1] + } else { + name = path.Base(uri) + fmt.Printf("Setting VNFS name: %s\n", name) + } + + if container.ValidName(name) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", name) + os.Exit(1) + } + + fullPath := container.SourceDir(name) + + if util.IsDir(fullPath) == true { + if SetForce == true { + wwlog.Printf(wwlog.WARN, "Overwriting existing VNFS\n") + err := os.RemoveAll(fullPath) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } else if SetUpdate == true { + wwlog.Printf(wwlog.WARN, "Updating existing VNFS\n") + } else { + wwlog.Printf(wwlog.ERROR, "VNFS Name exists, specify --force, --update, or choose a different name: %s\n", name) + os.Exit(1) + } + } + + err := container.PullURI(uri, name) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not pull image: %s\n", err) + os.Exit(1) + } + + if SetBuild == true { + fmt.Printf("Building container: %s\n", name) + container.Build(name, false) + } + + return nil +} diff --git a/internal/app/wwctl/container/pull/root.go b/internal/app/wwctl/container/pull/root.go new file mode 100644 index 00000000..84ed3d04 --- /dev/null +++ b/internal/app/wwctl/container/pull/root.go @@ -0,0 +1,28 @@ +package pull + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "pull", + Short: "Pull Source OCI VNFS images", + Long: "Pull Source OCI VNFS images ", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), + } + SetForce bool + SetUpdate bool + SetBuild bool +) + +func init() { + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force overwrite of an existing container") + baseCmd.PersistentFlags().BoolVarP(&SetUpdate, "update", "u", false, "Update and overwrite an existing container") + baseCmd.PersistentFlags().BoolVarP(&SetBuild, "build", "b", false, "Build container when after pulling") + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/container/root.go b/internal/app/wwctl/container/root.go new file mode 100644 index 00000000..3044c66a --- /dev/null +++ b/internal/app/wwctl/container/root.go @@ -0,0 +1,29 @@ +package container + +import ( + "github.com/hpcng/warewulf/internal/app/wwctl/container/build" + "github.com/hpcng/warewulf/internal/app/wwctl/container/list" + "github.com/hpcng/warewulf/internal/app/wwctl/container/pull" + "github.com/spf13/cobra" +) + +var ( + baseCmd = &cobra.Command{ + Use: "container", + Short: "VNFS image management", + Long: "Virtual Node File System (VNFS) image management", + } + test bool +) + +func init() { + baseCmd.AddCommand(build.GetCommand()) + baseCmd.AddCommand(list.GetCommand()) + baseCmd.AddCommand(pull.GetCommand()) + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} From 287395f85216e72ba194a3d57dfbfce2c9ec13e7 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 01:16:21 -0800 Subject: [PATCH 5/9] Hoping this fixes my .gitignore issues --- .gitignore | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index aad35f33..02198660 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ # project data -.idea -.tools -vendor/ +/.idea +/.tools +/vendor/ # binaries -warewulfd -wwbuild -wwclient -wwctl +/warewulfd +/wwbuild +/wwclient +/wwctl From 3041fcac6b13ae0ba99bf0627b884676b65d5a46 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 09:33:36 -0800 Subject: [PATCH 6/9] Added `wwctl container exec ...` feature --- .../app/wwctl/container/exec/child/main.go | 48 +++++++++++++++++++ .../app/wwctl/container/exec/child/root.go | 21 ++++++++ internal/app/wwctl/container/exec/main.go | 40 ++++++++++++++++ .../app/wwctl/container/exec/non-linux.go | 14 ++++++ internal/app/wwctl/container/exec/root.go | 26 ++++++++++ internal/app/wwctl/container/root.go | 4 +- 6 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 internal/app/wwctl/container/exec/child/main.go create mode 100644 internal/app/wwctl/container/exec/child/root.go create mode 100644 internal/app/wwctl/container/exec/main.go create mode 100644 internal/app/wwctl/container/exec/non-linux.go create mode 100644 internal/app/wwctl/container/exec/root.go diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go new file mode 100644 index 00000000..e718c0bc --- /dev/null +++ b/internal/app/wwctl/container/exec/child/main.go @@ -0,0 +1,48 @@ +// + build linux + +package child + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" + "syscall" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + + if os.Getpid() != 1 { + wwlog.Printf(wwlog.ERROR, "PID is not 1: %d\n", os.Getpid()) + os.Exit(1) + } + + containerName := args[0] + + if container.ValidSource(containerName) == false { + wwlog.Printf(wwlog.ERROR, "Unknown Warewulf container: %s\n", containerName) + os.Exit(1) + } + + syscall.Mount("", "/", "", syscall.MS_PRIVATE, "") + + containerPath := container.RootFsDir(containerName) + + syscall.Chroot(containerPath) + os.Chdir("/") + + syscall.Mount("rootfs", "rootfs", "", syscall.MS_BIND, "") + syscall.Mount("/proc", "/proc", "proc", 0, "") + + ps1string := fmt.Sprintf("[%s] Warewulf> ", containerName) + os.Setenv("PS1", ps1string) + + err := syscall.Exec(args[1], args[1:], os.Environ()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + + return nil +} diff --git a/internal/app/wwctl/container/exec/child/root.go b/internal/app/wwctl/container/exec/child/root.go new file mode 100644 index 00000000..6df31ff4 --- /dev/null +++ b/internal/app/wwctl/container/exec/child/root.go @@ -0,0 +1,21 @@ +package child + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "__child", + Hidden: true, + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), + } +) + +func init() { + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/container/exec/main.go b/internal/app/wwctl/container/exec/main.go new file mode 100644 index 00000000..23db6553 --- /dev/null +++ b/internal/app/wwctl/container/exec/main.go @@ -0,0 +1,40 @@ +// +build linux + +package exec + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" + "os/exec" + "syscall" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + + containerName := args[0] + + if container.ValidSource(containerName) == false { + wwlog.Printf(wwlog.ERROR, "Unknown Warewulf container: %s\n", containerName) + os.Exit(1) + } + + c := exec.Command("/proc/self/exe", append([]string{"container", "exec", "__child"}, args...)...) + + //c := exec.Command("/bin/sh") + c.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: syscall.CLONE_NEWUTS | syscall.CLONE_NEWPID | syscall.CLONE_NEWNS, + } + c.Stdin = os.Stdin + c.Stdout = os.Stdout + c.Stderr = os.Stderr + + if err := c.Run(); err != nil { + fmt.Println("ERROR", err) + os.Exit(1) + } + + return nil +} diff --git a/internal/app/wwctl/container/exec/non-linux.go b/internal/app/wwctl/container/exec/non-linux.go new file mode 100644 index 00000000..a391ec15 --- /dev/null +++ b/internal/app/wwctl/container/exec/non-linux.go @@ -0,0 +1,14 @@ +// +build !linux + +package exec + +import ( + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + wwlog.Printf(wwlog.ERROR, "This command does not work on non-Linux hosts\n") + + return nil +} diff --git a/internal/app/wwctl/container/exec/root.go b/internal/app/wwctl/container/exec/root.go new file mode 100644 index 00000000..049ca014 --- /dev/null +++ b/internal/app/wwctl/container/exec/root.go @@ -0,0 +1,26 @@ +package exec + +import ( + "github.com/hpcng/warewulf/internal/app/wwctl/container/exec/child" + "github.com/spf13/cobra" +) + +var ( + baseCmd = &cobra.Command{ + Use: "exec", + Short: "Spawn any command inside a Warewulf container", + Long: "Run a command inside a Warewulf container ", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), + } +) + +func init() { + baseCmd.AddCommand(child.GetCommand()) + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/container/root.go b/internal/app/wwctl/container/root.go index 3044c66a..9bda463a 100644 --- a/internal/app/wwctl/container/root.go +++ b/internal/app/wwctl/container/root.go @@ -2,6 +2,7 @@ package container import ( "github.com/hpcng/warewulf/internal/app/wwctl/container/build" + "github.com/hpcng/warewulf/internal/app/wwctl/container/exec" "github.com/hpcng/warewulf/internal/app/wwctl/container/list" "github.com/hpcng/warewulf/internal/app/wwctl/container/pull" "github.com/spf13/cobra" @@ -10,7 +11,7 @@ import ( var ( baseCmd = &cobra.Command{ Use: "container", - Short: "VNFS image management", + Short: "Container / VNFS image management", Long: "Virtual Node File System (VNFS) image management", } test bool @@ -20,6 +21,7 @@ func init() { baseCmd.AddCommand(build.GetCommand()) baseCmd.AddCommand(list.GetCommand()) baseCmd.AddCommand(pull.GetCommand()) + baseCmd.AddCommand(exec.GetCommand()) } From bbfe1023e9aaceec0311287a685f68c6007c038a Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 09:39:48 -0800 Subject: [PATCH 7/9] Add file for non-Linux placeholder. --- internal/app/wwctl/container/exec/child/main.go | 2 +- .../app/wwctl/container/exec/child/non-Linux.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 internal/app/wwctl/container/exec/child/non-Linux.go diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index e718c0bc..6e906fd8 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -1,4 +1,4 @@ -// + build linux +// +build linux package child diff --git a/internal/app/wwctl/container/exec/child/non-Linux.go b/internal/app/wwctl/container/exec/child/non-Linux.go new file mode 100644 index 00000000..c8cc1725 --- /dev/null +++ b/internal/app/wwctl/container/exec/child/non-Linux.go @@ -0,0 +1,14 @@ +// +build !linux + +package child + +import ( + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + wwlog.Printf(wwlog.ERROR, "This command does not work on non-Linux hosts\n") + + return nil +} From 9de97786488c5c33cdea8fdf934e692223f13459 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 10:00:04 -0800 Subject: [PATCH 8/9] Add bind points for /dev/ and /etc/resolv.conf into the container when exec'ing --- internal/app/wwctl/container/exec/child/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 6e906fd8..928de7a0 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -1,13 +1,15 @@ -// +build linux +// + build linux package child import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" + "path" "syscall" ) @@ -25,10 +27,15 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } - syscall.Mount("", "/", "", syscall.MS_PRIVATE, "") - containerPath := container.RootFsDir(containerName) + syscall.Mount("", "/", "", syscall.MS_PRIVATE, "") + syscall.Mount("/dev", path.Join(containerPath, "/dev"), "", syscall.MS_BIND, "") + + if util.IsFile(path.Join(containerPath, "/etc/resolv.conf")) == true { + syscall.Mount("/etc/resolv.conf", path.Join(containerPath, "/etc/resolv.conf"), "", syscall.MS_BIND, "") + } + syscall.Chroot(containerPath) os.Chdir("/") From 5c995f9d0f4b866893b867980fff336fe67f952e Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sun, 6 Dec 2020 22:29:25 -0800 Subject: [PATCH 9/9] Added `Init` to datastructure, fixups to container interface, and --setdefault to kernel and container build --- Makefile | 2 +- internal/app/wwctl/container/build/main.go | 27 +++++++++++++++++++ internal/app/wwctl/container/build/root.go | 2 ++ .../app/wwctl/container/exec/child/main.go | 2 +- internal/app/wwctl/kernel/build/main.go | 27 +++++++++++++++++++ internal/app/wwctl/kernel/build/root.go | 6 +++-- internal/app/wwctl/node/list/main.go | 1 + internal/app/wwctl/node/set/main.go | 10 +++++++ internal/app/wwctl/node/set/root.go | 2 ++ internal/app/wwctl/profile/list/main.go | 2 ++ internal/app/wwctl/profile/set/main.go | 11 ++++++++ internal/app/wwctl/profile/set/root.go | 3 +++ internal/pkg/container/vnfs.go | 7 +---- internal/pkg/node/constructors.go | 4 +++ internal/pkg/node/datastructure.go | 2 ++ internal/pkg/node/modifiers.go | 2 ++ internal/pkg/overlay/overlay.go | 2 ++ overlays/system/default/{init => init.ww} | 4 +-- 18 files changed, 104 insertions(+), 12 deletions(-) rename overlays/system/default/{init => init.ww} (95%) diff --git a/Makefile b/Makefile index 4675addf..d4e74189 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ files: all # cp -r tftpboot/* /var/lib/tftpboot/warewulf/ipxe/ # restorecon -r /var/lib/tftpboot/warewulf cp -r overlays /var/warewulf/ - chmod +x /var/warewulf/overlays/system/default/init + chmod +x /var/warewulf/overlays/system/default/init.ww mkdir -p /var/warewulf/overlays/system/default/warewulf/bin/ cp wwclient /var/warewulf/overlays/system/default/warewulf/bin/ diff --git a/internal/app/wwctl/container/build/main.go b/internal/app/wwctl/container/build/main.go index 81b6bb66..236bf712 100644 --- a/internal/app/wwctl/container/build/main.go +++ b/internal/app/wwctl/container/build/main.go @@ -1,7 +1,9 @@ package build import ( + "fmt" "github.com/hpcng/warewulf/internal/pkg/container" + "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" @@ -25,6 +27,31 @@ func CobraRunE(cmd *cobra.Command, args []string) error { container.Build(c, BuildForce) } + if SetDefault == true { + if len(containers) != 1 { + wwlog.Printf(wwlog.ERROR, "Can only set default for one container\n") + } else { + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + //TODO: Don't loop through profiles, instead have a nodeDB function that goes directly to the map + profiles, _ := nodeDB.FindAllProfiles() + for _, profile := range profiles { + wwlog.Printf(wwlog.DEBUG, "Looking for profile default: %s\n", profile.Id.Get()) + if profile.Id.Get() == "default" { + wwlog.Printf(wwlog.DEBUG, "Found profile default, setting container name to: %s\n", containers[0]) + profile.ContainerName.Set(containers[0]) + nodeDB.ProfileUpdate(profile) + } + } + nodeDB.Persist() + fmt.Printf("Set default profile to container: %s\n", containers[0]) + } + } + /* var nodes []node.NodeInfo set := make(map[string]int) diff --git a/internal/app/wwctl/container/build/root.go b/internal/app/wwctl/container/build/root.go index f174053d..49ab376b 100644 --- a/internal/app/wwctl/container/build/root.go +++ b/internal/app/wwctl/container/build/root.go @@ -12,11 +12,13 @@ var ( } BuildForce bool BuildAll bool + SetDefault bool ) func init() { baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "(re)Build all VNFS images for all nodes") baseCmd.PersistentFlags().BoolVarP(&BuildForce, "force", "f", false, "Force rebuild, even if it isn't necessary") + baseCmd.PersistentFlags().BoolVar(&SetDefault, "setdefault", false, "Set this container for the default profile") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/container/exec/child/main.go b/internal/app/wwctl/container/exec/child/main.go index 928de7a0..64222022 100644 --- a/internal/app/wwctl/container/exec/child/main.go +++ b/internal/app/wwctl/container/exec/child/main.go @@ -1,4 +1,4 @@ -// + build linux +// +build linux package child diff --git a/internal/app/wwctl/kernel/build/main.go b/internal/app/wwctl/kernel/build/main.go index 1e1c474d..39ac088f 100644 --- a/internal/app/wwctl/kernel/build/main.go +++ b/internal/app/wwctl/kernel/build/main.go @@ -1,7 +1,9 @@ package build import ( + "fmt" "github.com/hpcng/warewulf/internal/pkg/kernel" + "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" @@ -17,6 +19,31 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } + if SetDefault == true { + if len(args) != 1 { + wwlog.Printf(wwlog.ERROR, "Can only set default for one kernel version\n") + } else { + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + //TODO: Don't loop through profiles, instead have a nodeDB function that goes directly to the map + profiles, _ := nodeDB.FindAllProfiles() + for _, profile := range profiles { + wwlog.Printf(wwlog.DEBUG, "Looking for profile default: %s\n", profile.Id.Get()) + if profile.Id.Get() == "default" { + wwlog.Printf(wwlog.DEBUG, "Found profile default, setting kernel version to: %s\n", args[0]) + profile.KernelVersion.Set(args[0]) + nodeDB.ProfileUpdate(profile) + } + } + nodeDB.Persist() + fmt.Printf("Set default kernel version to: %s\n", args[0]) + } + } + /* var nodes []node.NodeInfo set := make(map[string]int) diff --git a/internal/app/wwctl/kernel/build/root.go b/internal/app/wwctl/kernel/build/root.go index 82f04fdf..9e873996 100644 --- a/internal/app/wwctl/kernel/build/root.go +++ b/internal/app/wwctl/kernel/build/root.go @@ -12,13 +12,15 @@ var ( RunE: CobraRunE, Args: cobra.MinimumNArgs(1), } - BuildAll bool - ByNode bool + BuildAll bool + ByNode bool + SetDefault bool ) func init() { baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "Build all overlays (runtime and system)") baseCmd.PersistentFlags().BoolVarP(&ByNode, "node", "n", false, "Build overlay for a particular node(s)") + baseCmd.PersistentFlags().BoolVar(&SetDefault, "setdefault", false, "Set this kernel for the default profile") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/node/list/main.go b/internal/app/wwctl/node/list/main.go index b2103686..4150db2d 100644 --- a/internal/app/wwctl/node/list/main.go +++ b/internal/app/wwctl/node/list/main.go @@ -56,6 +56,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "RuntimeOverlay", node.RuntimeOverlay.Source(), node.RuntimeOverlay.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "SystemOverlay", node.SystemOverlay.Source(), node.SystemOverlay.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "Ipxe", node.Ipxe.Source(), node.Ipxe.Print()) + fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "Init", node.Init.Source(), node.Init.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "IpmiIpaddr", node.IpmiIpaddr.Source(), node.IpmiIpaddr.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "IpmiNetmask", node.IpmiNetmask.Source(), node.IpmiNetmask.Print()) fmt.Printf("%-20s %-18s %-12s %s\n", node.Id.Get(), "IpmiUserName", node.IpmiUserName.Source(), node.IpmiUserName.Print()) diff --git a/internal/app/wwctl/node/set/main.go b/internal/app/wwctl/node/set/main.go index a6dba01b..942620d8 100644 --- a/internal/app/wwctl/node/set/main.go +++ b/internal/app/wwctl/node/set/main.go @@ -84,6 +84,16 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } } + if SetInit != "" { + wwlog.Printf(wwlog.VERBOSE, "Node: %s, Setting init command to: %s\n", n.Id.Get(), SetInit) + + n.Init.Set(SetInit) + err := nodeDB.NodeUpdate(n) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } if SetKernel != "" { wwlog.Printf(wwlog.VERBOSE, "Node: %s, Setting kernel to: %s\n", n.Id.Get(), SetKernel) diff --git a/internal/app/wwctl/node/set/root.go b/internal/app/wwctl/node/set/root.go index 24cb0f73..c0eff8c5 100644 --- a/internal/app/wwctl/node/set/root.go +++ b/internal/app/wwctl/node/set/root.go @@ -31,6 +31,7 @@ var ( SetAddProfile []string SetDelProfile []string SetForce bool + SetInit string ) func init() { @@ -39,6 +40,7 @@ func init() { baseCmd.PersistentFlags().StringVarP(&SetKernel, "kernel", "K", "", "Set Kernel version for nodes") baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "c", "", "Set the node's cluster group") baseCmd.PersistentFlags().StringVarP(&SetIpxe, "ipxe", "P", "", "Set the node's iPXE template name") + baseCmd.PersistentFlags().StringVarP(&SetInit, "init", "i", "", "Define the init process to boot the container") baseCmd.PersistentFlags().StringVarP(&SetRuntimeOverlay, "runtime", "R", "", "Set the node's runtime overlay") baseCmd.PersistentFlags().StringVarP(&SetSystemOverlay, "system", "S", "", "Set the node's system overlay") baseCmd.PersistentFlags().StringVar(&SetIpmiIpaddr, "ipmi", "", "Set the node's IPMI IP address") diff --git a/internal/app/wwctl/profile/list/main.go b/internal/app/wwctl/profile/list/main.go index 1c4e8762..e4f2e288 100644 --- a/internal/app/wwctl/profile/list/main.go +++ b/internal/app/wwctl/profile/list/main.go @@ -40,6 +40,8 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "ContainerName", profile.ContainerName.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "KernelVersion", profile.KernelVersion.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "KernelArgs", profile.KernelArgs.Print()) + fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "Init", profile.Init.Print()) + fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "RuntimeOverlay", profile.RuntimeOverlay.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "SystemOverlay", profile.SystemOverlay.Print()) fmt.Printf("%-20s %-18s %s\n", profile.Id.Get(), "Ipxe", profile.Ipxe.Print()) diff --git a/internal/app/wwctl/profile/set/main.go b/internal/app/wwctl/profile/set/main.go index c146574b..f0573fe6 100644 --- a/internal/app/wwctl/profile/set/main.go +++ b/internal/app/wwctl/profile/set/main.go @@ -98,6 +98,17 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } } + if SetInit != "" { + wwlog.Printf(wwlog.VERBOSE, "Profile: %s, Setting init command to: %s\n", p.Id, SetInit) + + p.Init.Set(SetInit) + err := nodeDB.ProfileUpdate(p) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } + if SetKernel != "" { wwlog.Printf(wwlog.VERBOSE, "Profile: %s, Setting Kernel version to: %s\n", p.Id, SetKernel) diff --git a/internal/app/wwctl/profile/set/root.go b/internal/app/wwctl/profile/set/root.go index cd610b94..8fc2f997 100644 --- a/internal/app/wwctl/profile/set/root.go +++ b/internal/app/wwctl/profile/set/root.go @@ -27,6 +27,7 @@ var ( SetGateway string SetHwaddr string SetNetDevDel bool + SetInit string ) func init() { @@ -35,6 +36,8 @@ func init() { baseCmd.PersistentFlags().StringVarP(&SetKernel, "kernel", "K", "", "Set Kernel version for nodes") baseCmd.PersistentFlags().StringVarP(&SetClusterName, "cluster", "c", "", "Set the node's cluster group") baseCmd.PersistentFlags().StringVarP(&SetIpxe, "ipxe", "P", "", "Set the node's iPXE template name") + baseCmd.PersistentFlags().StringVarP(&SetInit, "init", "i", "", "Define the init process to boot the container") + baseCmd.PersistentFlags().StringVarP(&SetRuntimeOverlay, "runtime", "R", "", "Set the node's runtime overlay") baseCmd.PersistentFlags().StringVarP(&SetSystemOverlay, "system", "S", "", "Set the node's system overlay") baseCmd.PersistentFlags().StringVar(&SetIpmiNetmask, "ipminetmask", "", "Set the node's IPMI netmask") diff --git a/internal/pkg/container/vnfs.go b/internal/pkg/container/vnfs.go index 0a3e6991..e86b8e45 100644 --- a/internal/pkg/container/vnfs.go +++ b/internal/pkg/container/vnfs.go @@ -77,12 +77,7 @@ func ValidSource(name string) bool { } if util.IsDir(fullPath) == false { - wwlog.Printf(wwlog.VERBOSE, "Location is not a VNFS source directory: %s\n", name) - return false - } - - if util.IsFile(path.Join(fullPath, "/sbin/init")) == false { - wwlog.Printf(wwlog.VERBOSE, "VNFS Source does not have a valid /sbin/init: %s\n", name) + wwlog.Printf(wwlog.WARN, "Location is not a VNFS source directory: %s\n", name) return false } diff --git a/internal/pkg/node/constructors.go b/internal/pkg/node/constructors.go index c7baedaa..1211b384 100644 --- a/internal/pkg/node/constructors.go +++ b/internal/pkg/node/constructors.go @@ -45,6 +45,7 @@ func (self *nodeYaml) FindAllNodes() ([]NodeInfo, error) { n.SystemOverlay.SetDefault("default") n.RuntimeOverlay.SetDefault("default") n.Ipxe.SetDefault("default") + n.Init.SetDefault("/sbin/init") fullname := strings.SplitN(nodename, ".", 2) if len(fullname) > 1 { @@ -64,6 +65,7 @@ func (self *nodeYaml) FindAllNodes() ([]NodeInfo, error) { n.KernelArgs.Set(node.KernelArgs) n.ClusterName.Set(node.ClusterName) n.Ipxe.Set(node.Ipxe) + n.Init.Set(node.Init) n.IpmiIpaddr.Set(node.IpmiIpaddr) n.IpmiNetmask.Set(node.IpmiNetmask) n.IpmiUserName.Set(node.IpmiUserName) @@ -101,6 +103,7 @@ func (self *nodeYaml) FindAllNodes() ([]NodeInfo, error) { n.KernelVersion.SetAlt(self.NodeProfiles[p].KernelVersion, pstring) n.KernelArgs.SetAlt(self.NodeProfiles[p].KernelArgs, pstring) n.Ipxe.SetAlt(self.NodeProfiles[p].Ipxe, pstring) + n.Init.SetAlt(self.NodeProfiles[p].Init, pstring) n.IpmiIpaddr.SetAlt(self.NodeProfiles[p].IpmiIpaddr, pstring) n.IpmiNetmask.SetAlt(self.NodeProfiles[p].IpmiNetmask, pstring) n.IpmiUserName.SetAlt(self.NodeProfiles[p].IpmiUserName, pstring) @@ -142,6 +145,7 @@ func (self *nodeYaml) FindAllProfiles() ([]NodeInfo, error) { p.Comment.Set(profile.Comment) p.ContainerName.Set(profile.ContainerName) p.Ipxe.Set(profile.Ipxe) + p.Init.Set(profile.Init) p.KernelVersion.Set(profile.KernelVersion) p.KernelArgs.Set(profile.KernelArgs) p.IpmiNetmask.Set(profile.IpmiNetmask) diff --git a/internal/pkg/node/datastructure.go b/internal/pkg/node/datastructure.go index cd7ea2eb..d4e8077e 100644 --- a/internal/pkg/node/datastructure.go +++ b/internal/pkg/node/datastructure.go @@ -29,6 +29,7 @@ type NodeConf struct { IpmiNetmask string `yaml:"ipmi netmask,omitempty"` RuntimeOverlay string `yaml:"runtime overlay files,omitempty"` SystemOverlay string `yaml:"system overlay files,omitempty"` + Init string `yaml:"init,omitempty"` Profiles []string `yaml:"profiles,omitempty"` NetDevs map[string]*NetDevs `yaml:"network devices,omitempty"` } @@ -70,6 +71,7 @@ type NodeInfo struct { IpmiPassword Entry RuntimeOverlay Entry SystemOverlay Entry + Init Entry //TODO: Finish adding this... Profiles []string GroupProfiles []string NetDevs map[string]*NetDevEntry diff --git a/internal/pkg/node/modifiers.go b/internal/pkg/node/modifiers.go index 9b5773f4..9905f28c 100644 --- a/internal/pkg/node/modifiers.go +++ b/internal/pkg/node/modifiers.go @@ -57,6 +57,7 @@ func (self *nodeYaml) NodeUpdate(node NodeInfo) error { self.Nodes[nodeID].ContainerName = node.ContainerName.GetReal() self.Nodes[nodeID].ClusterName = node.ClusterName.GetReal() self.Nodes[nodeID].Ipxe = node.Ipxe.GetReal() + self.Nodes[nodeID].Init = node.Init.GetReal() self.Nodes[nodeID].KernelVersion = node.KernelVersion.GetReal() self.Nodes[nodeID].KernelArgs = node.KernelArgs.GetReal() self.Nodes[nodeID].IpmiIpaddr = node.IpmiIpaddr.GetReal() @@ -127,6 +128,7 @@ func (self *nodeYaml) ProfileUpdate(profile NodeInfo) error { self.NodeProfiles[profileID].Comment = profile.Comment.GetReal() self.NodeProfiles[profileID].ContainerName = profile.ContainerName.GetReal() self.NodeProfiles[profileID].Ipxe = profile.Ipxe.GetReal() + self.NodeProfiles[profileID].Init = profile.Init.GetReal() self.NodeProfiles[profileID].ClusterName = profile.ClusterName.GetReal() self.NodeProfiles[profileID].KernelVersion = profile.KernelVersion.GetReal() self.NodeProfiles[profileID].KernelArgs = profile.KernelArgs.GetReal() diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 3ac359a4..d133aa14 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -22,6 +22,7 @@ type TemplateStruct struct { Hostname string GroupName string Container string + Init string IpmiIpaddr string IpmiNetmask string IpmiUserName string @@ -132,6 +133,7 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { t.Id = n.Id.Get() t.Hostname = n.Id.Get() t.Container = n.ContainerName.Get() + t.Init = n.Init.Get() t.IpmiIpaddr = n.IpmiIpaddr.Get() t.IpmiNetmask = n.IpmiNetmask.Get() t.IpmiUserName = n.IpmiUserName.Get() diff --git a/overlays/system/default/init b/overlays/system/default/init.ww similarity index 95% rename from overlays/system/default/init rename to overlays/system/default/init.ww index 3a21f341..547912b3 100755 --- a/overlays/system/default/init +++ b/overlays/system/default/init.ww @@ -33,8 +33,8 @@ mount -t devtmpfs devtmpfs /dev nohup /warewulf/bin/wwclient >/var/log/wwclient.log 2>&1