From 0095b5562402591407cbcbdb374e7fda62bf4ccb Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Sat, 5 Dec 2020 16:43:30 -0800 Subject: [PATCH] Updates around VNFS restructure and kernel restructure. More coming. --- internal/app/warewulfd/response/vnfs.go | 13 +- internal/app/wwctl/kernel/build/main.go | 99 +++++---- internal/app/wwctl/kernel/build/root.go | 17 +- internal/app/wwctl/kernel/list/main.go | 75 ++++--- internal/app/wwctl/node/set/main.go | 18 ++ internal/app/wwctl/node/set/root.go | 2 + internal/app/wwctl/profile/set/main.go | 19 ++ internal/app/wwctl/profile/set/root.go | 2 + internal/app/wwctl/ready/main.go | 12 +- internal/app/wwctl/vnfs/build/main.go | 107 +++++----- internal/app/wwctl/vnfs/list/main.go | 100 +++++---- internal/app/wwctl/vnfs/list/root.go | 10 +- internal/app/wwctl/vnfs/pull/main.go | 54 +++++ internal/app/wwctl/vnfs/pull/root.go | 26 +++ internal/app/wwctl/vnfs/root.go | 10 +- internal/pkg/config/config.go | 84 +------- internal/pkg/kernel/kernel.go | 64 +++++- internal/pkg/oci/cache.go | 10 +- internal/pkg/oci/puller.go | 16 +- internal/pkg/overlay/funcmap.go | 13 +- internal/pkg/util/util.go | 4 +- internal/pkg/vnfs/build.go | 54 +++++ internal/pkg/vnfs/pull.go | 45 +++++ internal/pkg/vnfs/vnfs.go | 189 +++++------------- .../pkg/{vnfs => vnfs_old}/containerdir.go | 0 internal/pkg/{vnfs => vnfs_old}/docker.go | 0 internal/pkg/{vnfs => vnfs_old}/util.go | 0 internal/pkg/vnfs_old/vnfs.go | 173 ++++++++++++++++ 28 files changed, 791 insertions(+), 425 deletions(-) create mode 100644 internal/app/wwctl/vnfs/pull/main.go create mode 100644 internal/app/wwctl/vnfs/pull/root.go create mode 100644 internal/pkg/vnfs/build.go create mode 100644 internal/pkg/vnfs/pull.go rename internal/pkg/{vnfs => vnfs_old}/containerdir.go (100%) rename internal/pkg/{vnfs => vnfs_old}/docker.go (100%) rename internal/pkg/{vnfs => vnfs_old}/util.go (100%) create mode 100644 internal/pkg/vnfs_old/vnfs.go diff --git a/internal/app/warewulfd/response/vnfs.go b/internal/app/warewulfd/response/vnfs.go index 5ef72c0e..38ac1c60 100644 --- a/internal/app/warewulfd/response/vnfs.go +++ b/internal/app/warewulfd/response/vnfs.go @@ -16,19 +16,14 @@ func VnfsSend(w http.ResponseWriter, req *http.Request) { } if node.Vnfs.Defined() == true { + vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) - v, err := vnfs.Load(node.Vnfs.Get()) - if err != nil { - w.WriteHeader(503) - log.Printf("ERROR: Could not load VNFS: %s\n", node.Id.Get()) - return - } - - err = sendFile(w, v.Image, node.Id.Get()) + err = sendFile(w, vnfsImage, node.Id.Get()) if err != nil { log.Printf("ERROR1: %s\n", err) + w.WriteHeader(503) } else { - log.Printf("SEND: %15s: %s\n", node.Id.Get(), v.Image) + log.Printf("SEND: %15s: %s\n", node.Id.Get(), vnfsImage) } } else { w.WriteHeader(503) diff --git a/internal/app/wwctl/kernel/build/main.go b/internal/app/wwctl/kernel/build/main.go index 25d0ef01..1e1c474d 100644 --- a/internal/app/wwctl/kernel/build/main.go +++ b/internal/app/wwctl/kernel/build/main.go @@ -2,66 +2,77 @@ package build import ( "github.com/hpcng/warewulf/internal/pkg/kernel" - "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" "os" ) func CobraRunE(cmd *cobra.Command, args []string) error { - var nodes []node.NodeInfo - set := make(map[string]int) - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) + for _, arg := range args { + err := kernel.Build(arg) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Failed building kernel: %s\n", err) + os.Exit(1) + } } - if len(args) == 1 && ByNode == true { - var err error - nodes, err = n.SearchByName(args[0]) + /* + var nodes []node.NodeInfo + set := make(map[string]int) + + n, err := node.New() if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) os.Exit(1) } - for _, node := range nodes { - if node.KernelVersion.Defined() == true { - set[node.KernelVersion.Get()]++ + if len(args) == 1 && ByNode == true { + var err error + nodes, err = n.SearchByName(args[0]) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + os.Exit(1) + } + + for _, node := range nodes { + if node.KernelVersion.Defined() == true { + set[node.KernelVersion.Get()]++ + } + } + + } else if BuildAll == true { + var err error + nodes, err = n.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) + os.Exit(1) + } + + for _, node := range nodes { + wwlog.Printf(wwlog.DEBUG, "evaluating node/kernel: %s/%s\n", node.Id.Get(), node.KernelVersion.Get()) + if node.KernelVersion.Defined() == true { + set[node.KernelVersion.Get()]++ + } + } + + } else if len(args) == 1 { + set[args[0]]++ + } else { + cmd.Usage() + os.Exit(1) + } + + for k := range set { + wwlog.Printf(wwlog.INFO, "Building kernel: %s\n", k) + err := kernel.Build(k) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) } } - } else if BuildAll == true { - var err error - nodes, err = n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) - os.Exit(1) - } - - for _, node := range nodes { - wwlog.Printf(wwlog.DEBUG, "evaluating node/kernel: %s/%s\n", node.Id.Get(), node.KernelVersion.Get()) - if node.KernelVersion.Defined() == true { - set[node.KernelVersion.Get()]++ - } - } - - } else if len(args) == 1 { - set[args[0]]++ - } else { - cmd.Usage() - os.Exit(1) - } - - for k := range set { - wwlog.Printf(wwlog.INFO, "Building kernel: %s\n", k) - err := kernel.Build(k) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } + */ return nil } diff --git a/internal/app/wwctl/kernel/build/root.go b/internal/app/wwctl/kernel/build/root.go index 5bce78c6..82f04fdf 100644 --- a/internal/app/wwctl/kernel/build/root.go +++ b/internal/app/wwctl/kernel/build/root.go @@ -1,20 +1,19 @@ package build import ( -"github.com/spf13/cobra" + "github.com/spf13/cobra" ) var ( baseCmd = &cobra.Command{ - Use: "build (kernel version | node search pattern)", - Short: "Kernel Image Build", - Long: "Build kernel images", - RunE: CobraRunE, - Args: cobra.RangeArgs(0,1), - + Use: "build (kernel version | node search pattern)", + Short: "Kernel Image Build", + Long: "Build kernel images", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), } BuildAll bool - ByNode bool + ByNode bool ) func init() { @@ -25,4 +24,4 @@ func init() { // GetRootCommand returns the root cobra.Command for the application. func GetCommand() *cobra.Command { return baseCmd -} \ No newline at end of file +} diff --git a/internal/app/wwctl/kernel/list/main.go b/internal/app/wwctl/kernel/list/main.go index 87b25e76..4b2132c7 100644 --- a/internal/app/wwctl/kernel/list/main.go +++ b/internal/app/wwctl/kernel/list/main.go @@ -2,17 +2,22 @@ package list import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" - "io/ioutil" "os" - "path" - "strings" ) func CobraRunE(cmd *cobra.Command, args []string) error { + + kernels, err := kernel.ListKernels() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + nconfig, _ := node.New() nodes, _ := nconfig.FindAllNodes() nodemap := make(map[string]int) @@ -21,31 +26,53 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodemap[n.KernelVersion.Get()]++ } - images, _ := ioutil.ReadDir(config.KernelParentDir()) + fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") + for _, k := range kernels { + image := kernel.KernelImage(k) - fmt.Printf("%-38s %-16s %-16s %s\n", "KERNEL VERSION", "KERNEL SIZE(k)", "KMODS SIZE(k)", "NODES") - fmt.Println(strings.Repeat("=", 80)) + if nodemap[k] == 0 { + nodemap[k] = 0 + } + fmt.Printf("%-35s %-6t %-6d\n", k, util.IsFile(image), nodemap[k]) - for _, file := range images { - if util.IsDir(path.Join(config.KernelParentDir(), file.Name())) { - var kernel_size int64 - var kmods_size int64 - if util.IsFile(config.KernelImage(file.Name())) { - s, _ := os.Stat(config.KernelImage(file.Name())) - kernel_size = s.Size() / 1024 - } - if util.IsFile(config.KmodsImage(file.Name())) { - s, _ := os.Stat(config.KmodsImage(file.Name())) - kmods_size = s.Size() / 1024 - } + } - if nodemap[file.Name()] > 0 { - fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, nodemap[file.Name()]) - } else { - fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, 0) + /* + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.KernelVersion.Get()]++ + } + + images, _ := ioutil.ReadDir(config.KernelParentDir()) + + fmt.Printf("%-38s %-16s %-16s %s\n", "KERNEL VERSION", "KERNEL SIZE(k)", "KMODS SIZE(k)", "NODES") + fmt.Println(strings.Repeat("=", 80)) + + for _, file := range images { + if util.IsDir(path.Join(config.KernelParentDir(), file.Name())) { + var kernel_size int64 + var kmods_size int64 + if util.IsFile(config.KernelImage(file.Name())) { + s, _ := os.Stat(config.KernelImage(file.Name())) + kernel_size = s.Size() / 1024 + } + if util.IsFile(config.KmodsImage(file.Name())) { + s, _ := os.Stat(config.KmodsImage(file.Name())) + kmods_size = s.Size() / 1024 + } + + if nodemap[file.Name()] > 0 { + fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, nodemap[file.Name()]) + } else { + fmt.Printf("%-38s %-16d %-16d %d\n", file.Name(), kernel_size, kmods_size, 0) + } } } - } + + */ return nil } diff --git a/internal/app/wwctl/node/set/main.go b/internal/app/wwctl/node/set/main.go index cfc40362..37b94e39 100644 --- a/internal/app/wwctl/node/set/main.go +++ b/internal/app/wwctl/node/set/main.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -43,6 +44,23 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } + if SetVnfs != "" { + if vnfs.ValidSource(SetVnfs) == true { + imageFile := vnfs.ImageFile(SetVnfs) + if util.IsFile(imageFile) == false { + wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } else { + wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } + for _, n := range nodes { wwlog.Printf(wwlog.VERBOSE, "Evaluating node: %s\n", n.Id.Get()) diff --git a/internal/app/wwctl/node/set/root.go b/internal/app/wwctl/node/set/root.go index d905accf..aa21b77c 100644 --- a/internal/app/wwctl/node/set/root.go +++ b/internal/app/wwctl/node/set/root.go @@ -30,6 +30,7 @@ var ( SetYes bool SetAddProfile []string SetDelProfile []string + SetForce bool ) func init() { @@ -57,6 +58,7 @@ func init() { baseCmd.PersistentFlags().BoolVarP(&SetNodeAll, "all", "a", false, "Set all nodes") baseCmd.PersistentFlags().BoolVarP(&SetYes, "yes", "y", false, "Set 'yes' to all questions asked") + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force configuration (even on error)") } diff --git a/internal/app/wwctl/profile/set/main.go b/internal/app/wwctl/profile/set/main.go index cc185f41..f4058894 100644 --- a/internal/app/wwctl/profile/set/main.go +++ b/internal/app/wwctl/profile/set/main.go @@ -3,6 +3,8 @@ package set import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -46,6 +48,23 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } + if SetVnfs != "" { + if vnfs.ValidSource(SetVnfs) == true { + imageFile := vnfs.ImageFile(SetVnfs) + if util.IsFile(imageFile) == false { + wwlog.Printf(wwlog.ERROR, "VNFS has not been built: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } else { + wwlog.Printf(wwlog.ERROR, "VNFS does not exist: %s\n", SetVnfs) + if SetForce == false { + os.Exit(1) + } + } + } + for _, p := range profiles { wwlog.Printf(wwlog.VERBOSE, "Modifying profile: %s\n", p.Id.Get()) diff --git a/internal/app/wwctl/profile/set/root.go b/internal/app/wwctl/profile/set/root.go index 14e3269e..d841534c 100644 --- a/internal/app/wwctl/profile/set/root.go +++ b/internal/app/wwctl/profile/set/root.go @@ -10,6 +10,7 @@ var ( RunE: CobraRunE, } SetAll bool + SetForce bool SetComment string SetVnfs string SetKernel string @@ -48,6 +49,7 @@ func init() { baseCmd.PersistentFlags().BoolVar(&SetNetDevDel, "netdel", false, "Delete the node's network device") baseCmd.PersistentFlags().BoolVarP(&SetAll, "all", "a", false, "Set all profiles") + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force configuration (even on error)") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/ready/main.go b/internal/app/wwctl/ready/main.go index af7e7ef0..1369b7fc 100644 --- a/internal/app/wwctl/ready/main.go +++ b/internal/app/wwctl/ready/main.go @@ -3,6 +3,7 @@ package ready import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/kernel" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" @@ -36,12 +37,13 @@ func CobraRunE(cmd *cobra.Command, args []string) error { status := true if node.Vnfs.Get() != "" { - v, _ := vnfs.Load(node.Vnfs.Get()) - if util.IsFile(v.Image) == true { + vnfsImage := vnfs.ImageFile(node.Vnfs.Get()) + + if util.IsFile(vnfsImage) == true { vnfs_good = true } else { status = false - wwlog.Printf(wwlog.VERBOSE, "VNFS not found: %s, %s\n", node.Id.Get(), v.Source) + wwlog.Printf(wwlog.VERBOSE, "VNFS not found: %s, %s\n", node.Id.Get(), vnfsImage) } } else { status = false @@ -49,13 +51,13 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } if node.KernelVersion.Get() != "" { - if util.IsFile(config.KernelImage(node.KernelVersion.Get())) == true { + if util.IsFile(kernel.KernelImage(node.KernelVersion.Get())) == true { kernel_good = true } else { status = false wwlog.Printf(wwlog.VERBOSE, "Node Kernel not found: %s, %s\n", node.Id.Get(), node.KernelVersion.Get()) } - if util.IsFile(config.KmodsImage(node.KernelVersion.Get())) == true { + if util.IsFile(kernel.KmodsImage(node.KernelVersion.Get())) == true { kmods_good = true } else { status = false diff --git a/internal/app/wwctl/vnfs/build/main.go b/internal/app/wwctl/vnfs/build/main.go index 5952aba4..fa2330e1 100644 --- a/internal/app/wwctl/vnfs/build/main.go +++ b/internal/app/wwctl/vnfs/build/main.go @@ -1,8 +1,6 @@ package build import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" @@ -10,59 +8,72 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { - var nodes []node.NodeInfo - set := make(map[string]int) - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - if len(args) == 1 && ByNode == true { - var err error - nodes, err = n.SearchByName(args[0]) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + for _, arg := range args { + if vnfs.ValidSource(arg) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name does not exist: %s\n", arg) os.Exit(1) } - for _, node := range nodes { - if node.Vnfs.Defined() == true { - set[node.Vnfs.Get()]++ + vnfs.Build(arg, BuildForce) + } + + /* + var nodes []node.NodeInfo + set := make(map[string]int) + + n, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + + if len(args) == 1 && ByNode == true { + var err error + nodes, err = n.SearchByName(args[0]) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not find nodes for search term: %s\n", args[0]) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + set[node.Vnfs.Get()]++ + } + } + + } else if BuildAll == true { + var err error + nodes, err = n.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) + os.Exit(1) + } + + for _, node := range nodes { + if node.Vnfs.Defined() == true { + wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) + set[node.Vnfs.Get()]++ + } + } + + } else if len(args) == 1 { + set[args[0]]++ + } else { + cmd.Usage() + os.Exit(1) + } + + for v := range set { + fmt.Printf("Building VNFS: %s\n", v) + err := vnfs.Build(v, BuildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) } } - } else if BuildAll == true { - var err error - nodes, err = n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get list of nodes: %s\n", err) - os.Exit(1) - } - - for _, node := range nodes { - if node.Vnfs.Defined() == true { - wwlog.Printf(wwlog.VERBOSE, "Adding VNFS to list: %s (%s)\n", node.Vnfs.Get(), node.Id.Get()) - set[node.Vnfs.Get()]++ - } - } - - } else if len(args) == 1 { - set[args[0]]++ - } else { - cmd.Usage() - os.Exit(1) - } - - for v := range set { - fmt.Printf("Building VNFS: %s\n", v) - err := vnfs.Build(v, BuildForce) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - } + */ return nil } diff --git a/internal/app/wwctl/vnfs/list/main.go b/internal/app/wwctl/vnfs/list/main.go index 212fcbdf..2499fbac 100644 --- a/internal/app/wwctl/vnfs/list/main.go +++ b/internal/app/wwctl/vnfs/list/main.go @@ -2,18 +2,22 @@ package list import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/spf13/cobra" - "io/ioutil" "os" - "path" - "strings" ) func CobraRunE(cmd *cobra.Command, args []string) error { + + sources, err := vnfs.ListSources() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + nconfig, _ := node.New() nodes, _ := nconfig.FindAllNodes() nodemap := make(map[string]int) @@ -22,42 +26,64 @@ func CobraRunE(cmd *cobra.Command, args []string) error { nodemap[n.Vnfs.Get()]++ } - images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) - - fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") - fmt.Println(strings.Repeat("=", 80)) - - for _, file := range images { - v, err := vnfs.Load(file.Name()) - if err == nil { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[v.Source] > 0 { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) - } else { - fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) - } + fmt.Printf("%-35s %-6s %-6s\n", "VNFS NAME", "BUILT", "NODES") + for _, source := range sources { + image := vnfs.ImageFile(source) + if nodemap[source] == 0 { + nodemap[source] = 0 } - continue + fmt.Printf("%-35s %-6t %-6d\n", source, util.IsFile(image), nodemap[source]) - if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { - var vnfs_size int64 - if util.IsFile(config.VnfsImage(file.Name())) { - s, _ := os.Stat(config.VnfsImage(file.Name())) - vnfs_size = s.Size() / 1024 - } - - if nodemap[file.Name()] > 0 { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) - } else { - fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) - } - } } + + /* + nconfig, _ := node.New() + nodes, _ := nconfig.FindAllNodes() + nodemap := make(map[string]int) + + for _, n := range nodes { + nodemap[n.Vnfs.Get()]++ + } + + images, _ := ioutil.ReadDir(config.VnfsImageParentDir()) + + fmt.Printf("%-38s %-16s %s\n", "VNFS Name", "VNFS SIZE(k)", "NODES") + fmt.Println(strings.Repeat("=", 80)) + + for _, file := range images { + v, err := vnfs.Load(file.Name()) + if err == nil { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[v.Source] > 0 { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, nodemap[v.Source]) + } else { + fmt.Printf("%-38s %-16d %d\n", v.Source, vnfs_size, 0) + } + + } + continue + + if util.IsDir(path.Join(config.VnfsImageParentDir(), file.Name())) { + var vnfs_size int64 + if util.IsFile(config.VnfsImage(file.Name())) { + s, _ := os.Stat(config.VnfsImage(file.Name())) + vnfs_size = s.Size() / 1024 + } + + if nodemap[file.Name()] > 0 { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, nodemap[file.Name()]) + } else { + fmt.Printf("%-38s %-16d %d\n", file.Name(), vnfs_size, 0) + } + } + } + */ + return nil } diff --git a/internal/app/wwctl/vnfs/list/root.go b/internal/app/wwctl/vnfs/list/root.go index 58662f4d..80216bb9 100644 --- a/internal/app/wwctl/vnfs/list/root.go +++ b/internal/app/wwctl/vnfs/list/root.go @@ -4,13 +4,13 @@ import "github.com/spf13/cobra" var ( baseCmd = &cobra.Command{ - Use: "list", - Short: "List VNFS images", - Long: "List VNFS images ", - RunE: CobraRunE, + Use: "list", + Short: "List VNFS images", + Long: "List VNFS images ", + RunE: CobraRunE, } SystemOverlay bool - BuildAll bool + BuildAll bool ) func init() { diff --git a/internal/app/wwctl/vnfs/pull/main.go b/internal/app/wwctl/vnfs/pull/main.go new file mode 100644 index 00000000..b85466fe --- /dev/null +++ b/internal/app/wwctl/vnfs/pull/main.go @@ -0,0 +1,54 @@ +package pull + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" + "path" +) + +func CobraRunE(cmd *cobra.Command, args []string) error { + var name string + uri := args[0] + + if len(args) == 2 { + name = args[1] + } else { + name = path.Base(uri) + fmt.Printf("Setting VNFS name: %s\n", name) + } + + if vnfs.ValidName(name) == false { + wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", name) + os.Exit(1) + } + + fullPath := vnfs.SourceDir(name) + + if util.IsDir(fullPath) == true { + if SetForce == true { + wwlog.Printf(wwlog.WARN, "Overwriting existing VNFS\n") + err := os.RemoveAll(fullPath) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + } else if SetUpdate == true { + wwlog.Printf(wwlog.WARN, "Updating existing VNFS\n") + } else { + wwlog.Printf(wwlog.ERROR, "VNFS Name exists, specify --force, --update, or choose a different name: %s\n", name) + os.Exit(1) + } + } + + err := vnfs.PullURI(uri, name) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not pull image: %s\n", err) + os.Exit(1) + } + + return nil +} diff --git a/internal/app/wwctl/vnfs/pull/root.go b/internal/app/wwctl/vnfs/pull/root.go new file mode 100644 index 00000000..93d88329 --- /dev/null +++ b/internal/app/wwctl/vnfs/pull/root.go @@ -0,0 +1,26 @@ +package pull + +import "github.com/spf13/cobra" + +var ( + baseCmd = &cobra.Command{ + Use: "pull", + Short: "Pull Source OCI VNFS images", + Long: "Pull Source OCI VNFS images ", + RunE: CobraRunE, + Args: cobra.MinimumNArgs(1), + } + SetForce bool + SetUpdate bool +) + +func init() { + baseCmd.PersistentFlags().BoolVarP(&SetForce, "force", "f", false, "Force overwrite of an existing VNFS") + baseCmd.PersistentFlags().BoolVarP(&SetUpdate, "update", "u", false, "Update and overwrite an existing VNFS") + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return baseCmd +} diff --git a/internal/app/wwctl/vnfs/root.go b/internal/app/wwctl/vnfs/root.go index 851d1d2b..0b10b5c2 100644 --- a/internal/app/wwctl/vnfs/root.go +++ b/internal/app/wwctl/vnfs/root.go @@ -3,14 +3,15 @@ package vnfs import ( "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/build" "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/list" + "github.com/hpcng/warewulf/internal/app/wwctl/vnfs/pull" "github.com/spf13/cobra" ) var ( baseCmd = &cobra.Command{ - Use: "vnfs", - Short: "VNFS image management", - Long: "Virtual Node File System (VNFS) image management", + Use: "vnfs", + Short: "VNFS image management", + Long: "Virtual Node File System (VNFS) image management", } test bool ) @@ -18,10 +19,11 @@ var ( func init() { baseCmd.AddCommand(build.GetCommand()) baseCmd.AddCommand(list.GetCommand()) + baseCmd.AddCommand(pull.GetCommand()) + } // GetRootCommand returns the root cobra.Command for the application. func GetCommand() *cobra.Command { return baseCmd } - diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 8586e396..5a189525 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -11,10 +11,6 @@ const ( LocalStateDir = "/var/warewulf" ) -func NodeConfig() string { - return fmt.Sprintf("%s/nodes.conf", LocalStateDir) -} - func OverlayDir() string { return fmt.Sprintf("%s/overlays/", LocalStateDir) } @@ -27,25 +23,13 @@ func RuntimeOverlayDir() string { return path.Join(OverlayDir(), "/runtime") } -func VnfsImageParentDir() string { - return fmt.Sprintf("%s/provision/vnfs/", LocalStateDir) -} - -func VnfsChrootParentDir() string { - return fmt.Sprintf("%s/chroot/", LocalStateDir) -} - -func KernelParentDir() string { - return fmt.Sprintf("%s/provision/kernel/", LocalStateDir) -} - func SystemOverlaySource(overlayName string) string { if overlayName == "" { wwlog.Printf(wwlog.ERROR, "System overlay name is not defined\n") return "" } - if util.TaintCheck(overlayName, "^[a-zA-Z0-9-._]+$") == false { + if util.ValidString(overlayName, "^[a-zA-Z0-9-._]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", overlayName) return "" } @@ -59,7 +43,7 @@ func RuntimeOverlaySource(overlayName string) string { return "" } - if util.TaintCheck(overlayName, "^[a-zA-Z0-9-._]+$") == false { + if util.ValidString(overlayName, "^[a-zA-Z0-9-._]+$") == false { wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", overlayName) return "" } @@ -67,41 +51,13 @@ func RuntimeOverlaySource(overlayName string) string { return path.Join(RuntimeOverlayDir(), overlayName) } -func KernelImage(kernelVersion string) string { - if kernelVersion == "" { - wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") - return "" - } - - if util.TaintCheck(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { - wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) - return "" - } - - return path.Join(KernelParentDir(), kernelVersion, "vmlinuz") -} - -func KmodsImage(kernelVersion string) string { - if kernelVersion == "" { - wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") - return "" - } - - if util.TaintCheck(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { - wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) - return "" - } - - return path.Join(KernelParentDir(), kernelVersion, "kmods.img") -} - func SystemOverlayImage(nodeName string) string { if nodeName == "" { wwlog.Printf(wwlog.ERROR, "Node name is not defined\n") return "" } - if util.TaintCheck(nodeName, "^[a-zA-Z0-9-._:]+$") == false { + if util.ValidString(nodeName, "^[a-zA-Z0-9-._:]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", nodeName) return "" } @@ -115,42 +71,10 @@ func RuntimeOverlayImage(nodeName string) string { return "" } - if util.TaintCheck(nodeName, "^[a-zA-Z0-9-._:]+$") == false { + if util.ValidString(nodeName, "^[a-zA-Z0-9-._:]+$") == false { wwlog.Printf(wwlog.ERROR, "System overlay name contains illegal characters: %s\n", nodeName) return "" } return fmt.Sprintf("%s/provision/overlays/runtime/%s.img", LocalStateDir, nodeName) } - -func VnfsImageDir(uri string) string { - if uri == "" { - wwlog.Printf(wwlog.ERROR, "VNFS URI is not defined\n") - return "" - } - - if util.TaintCheck(uri, "^[a-zA-Z0-9-._:]+$") == false { - wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", uri) - return "" - } - - return path.Join(VnfsImageParentDir(), uri) -} - -func VnfsImage(uri string) string { - return path.Join(VnfsImageDir(uri), "image") -} - -func VnfsChroot(uri string) string { - if uri == "" { - wwlog.Printf(wwlog.ERROR, "VNFS name is not defined\n") - return "" - } - - if util.TaintCheck(uri, "^[a-zA-Z0-9-._:]+$") == false { - wwlog.Printf(wwlog.ERROR, "VNFS name contains illegal characters: %s\n", uri) - return "" - } - - return path.Join(VnfsChrootParentDir(), uri) -} diff --git a/internal/pkg/kernel/kernel.go b/internal/pkg/kernel/kernel.go index 26109458..e59444f7 100644 --- a/internal/pkg/kernel/kernel.go +++ b/internal/pkg/kernel/kernel.go @@ -6,17 +6,75 @@ import ( "github.com/hpcng/warewulf/internal/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" + "io/ioutil" "os" "os/exec" "path" ) +func ParentDir() string { + return path.Join(config.LocalStateDir, "provision/kernel") +} + +func KernelImage(kernelVersion string) string { + if kernelVersion == "" { + wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") + return "" + } + + if util.ValidString(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { + wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) + return "" + } + + return path.Join(ParentDir(), kernelVersion, "vmlinuz") +} + +func KmodsImage(kernelVersion string) string { + if kernelVersion == "" { + wwlog.Printf(wwlog.ERROR, "Kernel Version is not defined\n") + return "" + } + + if util.ValidString(kernelVersion, "^[a-zA-Z0-9-._]+$") == false { + wwlog.Printf(wwlog.ERROR, "Runtime overlay name contains illegal characters: %s\n", kernelVersion) + return "" + } + + return path.Join(ParentDir(), kernelVersion, "kmods.img") +} + +func ListKernels() ([]string, error) { + var ret []string + + err := os.MkdirAll(ParentDir(), 0755) + if err != nil { + return ret, errors.New("Could not create Kernel parent directory: " + ParentDir()) + } + + wwlog.Printf(wwlog.DEBUG, "Searching for Kernel image directories: %s\n", ParentDir()) + + kernels, err := ioutil.ReadDir(ParentDir()) + if err != nil { + return ret, err + } + + for _, kernel := range kernels { + wwlog.Printf(wwlog.VERBOSE, "Found Kernel: %s\n", kernel.Name()) + + ret = append(ret, kernel.Name()) + + } + + return ret, nil +} + func Build(kernelVersion string) error { kernelImage := "/boot/vmlinuz-" + kernelVersion kernelDrivers := "/lib/modules/" + kernelVersion - kernelDestination := config.KernelImage(kernelVersion) - driversDestination := config.KmodsImage(kernelVersion) + kernelDestination := KernelImage(kernelVersion) + driversDestination := KmodsImage(kernelVersion) // Create the destination paths just in case it doesn't exist os.MkdirAll(path.Dir(kernelDestination), 0755) @@ -30,6 +88,7 @@ func Build(kernelVersion string) error { return errors.New("Could not locate kernel drivers: " + kernelDrivers) } + wwlog.Printf(wwlog.VERBOSE, "Setting up Kernel\n") if _, err := os.Stat(kernelImage); err == nil { err := util.CopyFile(kernelImage, kernelDestination) if err != nil { @@ -38,6 +97,7 @@ func Build(kernelVersion string) error { wwlog.Printf(wwlog.INFO, "%-45s: Done\n", "vmlinuz-"+kernelVersion) } + wwlog.Printf(wwlog.VERBOSE, "Building Kernel driver image\n") if _, err := os.Stat(kernelDrivers); err == nil { cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelDrivers, driversDestination) err := exec.Command("/bin/sh", "-c", cmd).Run() diff --git a/internal/pkg/oci/cache.go b/internal/pkg/oci/cache.go index 6aaa0828..369efaf4 100644 --- a/internal/pkg/oci/cache.go +++ b/internal/pkg/oci/cache.go @@ -79,15 +79,15 @@ func NewCache(opts ...CacheOpt) (*Cache, error) { } func (c *Cache) Pull(ctx context.Context, uri string, sysCtx *types.SystemContext) (string, error) { - p, err := newPuller( - optSetBlobCachePath(c.blobDir()), - optSetSystemContext(sysCtx), + p, err := NewPuller( + OptSetBlobCachePath(c.blobDir()), + OptSetSystemContext(sysCtx), ) if err != nil { return "", err } - id, err := p.generateID(ctx, uri) + id, err := p.GenerateID(ctx, uri) if err != nil { return "", err } @@ -107,7 +107,7 @@ func (c *Cache) Pull(ctx context.Context, uri string, sysCtx *types.SystemContex } // populate entry - if err := p.pull(ctx, uri, path); err != nil { + if err := p.Pull(ctx, uri, path); err != nil { // clean up entry on error os.RemoveAll(path) return "", err diff --git a/internal/pkg/oci/puller.go b/internal/pkg/oci/puller.go index 8b497ebf..db393024 100644 --- a/internal/pkg/oci/puller.go +++ b/internal/pkg/oci/puller.go @@ -23,21 +23,21 @@ import ( type pullerOpt func(*puller) error -func optSetBlobCachePath(path string) pullerOpt { +func OptSetBlobCachePath(path string) pullerOpt { return func(p *puller) error { p.blobCachePath = path return nil } } -func optSetTmpDirPath(path string) pullerOpt { +func OptSetTmpDirPath(path string) pullerOpt { return func(p *puller) error { p.tmpDirPath = path return nil } } -func optSetSystemContext(s *types.SystemContext) pullerOpt { +func OptSetSystemContext(s *types.SystemContext) pullerOpt { return func(p *puller) error { p.sysCtx = s return nil @@ -51,7 +51,7 @@ type puller struct { sysCtx *types.SystemContext } -func newPuller(opts ...pullerOpt) (*puller, error) { +func NewPuller(opts ...pullerOpt) (*puller, error) { p := &puller{ // default to a sensible value, but caller should set this with opts blobCachePath: filepath.Join(defaultCachePath, blobPrefix), @@ -93,8 +93,8 @@ func getReference(uri string) (types.ImageReference, error) { } } -// generateID stores and returns a unique identifier derived from the sha256sum of the image manifest -func (p *puller) generateID(ctx context.Context, uri string) (string, error) { +// GenerateID stores and returns a unique identifier derived from the sha256sum of the image manifest +func (p *puller) GenerateID(ctx context.Context, uri string) (string, error) { ref, err := getReference(uri) if err != nil { return "", fmt.Errorf("unable to parse uri: %v", err) @@ -114,7 +114,7 @@ func (p *puller) generateID(ctx context.Context, uri string) (string, error) { return p.id, nil } -func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { +func (p *puller) Pull(ctx context.Context, uri, dst string) (err error) { srcRef, err := getReference(uri) if err != nil { return fmt.Errorf("unable to parse uri: %v", err) @@ -135,7 +135,7 @@ func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { // copy to cache location _, err = copy.Image(ctx, policyCtx, cacheRef, srcRef, ©.Options{ ReportWriter: os.Stdout, - SourceCtx: p.sysCtx, + SourceCtx: p.sysCtx, }) if err != nil { return err diff --git a/internal/pkg/overlay/funcmap.go b/internal/pkg/overlay/funcmap.go index 3cf8e0b1..7843a4ae 100644 --- a/internal/pkg/overlay/funcmap.go +++ b/internal/pkg/overlay/funcmap.go @@ -1,7 +1,6 @@ package overlay import ( - "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" "io/ioutil" @@ -26,14 +25,14 @@ func templateVnfsFileInclude(vnfsname string, filepath string) string { return "" } - v, _ := vnfs.Load(vnfsname) - vnfsDir := v.Chroot - - if util.IsDir(vnfsDir) == false { - wwlog.Printf(wwlog.WARN, "Template requires VNFS (%s) to be imported: %s\n", vnfsname, filepath) + if vnfs.ValidSource(vnfsname) == false { + wwlog.Printf(wwlog.WARN, "Template required VNFS does not exist: %s\n", vnfsname) return "" } - wwlog.Printf(wwlog.DEBUG, "IncludeVnfs file from: %s/%s\n", vnfsDir, filepath) + + vnfsDir := vnfs.RootFsDir(vnfsname) + + wwlog.Printf(wwlog.DEBUG, "Including file from VNFS: %s:%s\n", vnfsDir, filepath) content, err := ioutil.ReadFile(path.Join(vnfsDir, filepath)) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index a768e04e..c342afd2 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -114,7 +114,7 @@ func IsFile(path string) bool { return false } -func TaintCheck(pattern string, expr string) bool { +func ValidString(pattern string, expr string) bool { if b, _ := regexp.MatchString(expr, pattern); b == true { return true } @@ -122,7 +122,7 @@ func TaintCheck(pattern string, expr string) bool { } func ValidateOrDie(message string, pattern string, expr string) { - if TaintCheck(pattern, expr) == false { + if ValidString(pattern, expr) == false { wwlog.Printf(wwlog.ERROR, "%s does not validate: '%s'\n", message, pattern) os.Exit(1) } diff --git a/internal/pkg/vnfs/build.go b/internal/pkg/vnfs/build.go new file mode 100644 index 00000000..101d6b7c --- /dev/null +++ b/internal/pkg/vnfs/build.go @@ -0,0 +1,54 @@ +package vnfs + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "os" + "os/exec" + "path" +) + +func Build(name string, buildForce bool) { + + rootfsPath := RootFsDir(name) + imagePath := ImageFile(name) + + if ValidSource(name) == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping (bad path)\n", name) + return + } + + if buildForce == false { + wwlog.Printf(wwlog.DEBUG, "Checking if there have been any updates to the VNFS directory\n") + if util.PathIsNewer(rootfsPath, imagePath) { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", name) + return + } + } + + wwlog.Printf(wwlog.DEBUG, "Making parent directory for: %s\n", name) + err := os.MkdirAll(path.Dir(imagePath), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + + wwlog.Printf(wwlog.DEBUG, "Making parent directory for: %s\n", rootfsPath) + err = os.MkdirAll(path.Dir(rootfsPath), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + + wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", rootfsPath, imagePath) + cmd := fmt.Sprintf("cd %s; find . | cpio --quiet -o -H newc | gzip -c > \"%s\"", rootfsPath, imagePath) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Failed building VNFS: %s\n", err) + return + } + + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", name) + +} diff --git a/internal/pkg/vnfs/pull.go b/internal/pkg/vnfs/pull.go new file mode 100644 index 00000000..f9fd75aa --- /dev/null +++ b/internal/pkg/vnfs/pull.go @@ -0,0 +1,45 @@ +package vnfs + +import ( + "context" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/errors" + "github.com/hpcng/warewulf/internal/pkg/oci" + "os" +) + +func PullURI(uri string, name string) error { + OciBlobCacheDir := config.LocalStateDir + "/oci/blobs" + + err := os.MkdirAll(OciBlobCacheDir, 0755) + if err != nil { + return err + } + + if ValidName(name) == false { + return errors.New("VNFS name contains illegal characters: " + name) + } + + fullPath := RootFsDir(name) + + err = os.MkdirAll(fullPath, 0755) + if err != nil { + return err + } + + p, err := oci.NewPuller( + oci.OptSetBlobCachePath(OciBlobCacheDir), + oci.OptSetSystemContext(nil), + ) + if err != nil { + return err + } + + p.GenerateID(context.Background(), uri) + + if err := p.Pull(context.Background(), uri, fullPath); err != nil { + return err + } + + return nil +} diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 32454fb9..ac40fc9e 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -5,169 +5,86 @@ import ( "github.com/hpcng/warewulf/internal/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "gopkg.in/yaml.v2" "io/ioutil" "os" "path" - "strings" ) -type VnfsObject struct { - Name string - Source string - Chroot string - Image string - Config string +func ValidName(name string) bool { + if util.ValidString(name, "^[a-zA-Z0-9.:-]+$") == false { + wwlog.Printf(wwlog.WARN, "VNFS name has illegal characters: %s\n", name) + return false + } + return true } -func Load(name string) (VnfsObject, error) { - var ret VnfsObject +func SourceParentDir() string { + return path.Join(config.LocalStateDir, "chroots") +} - if name == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") - } +func SourceDir(name string) string { + return path.Join(SourceParentDir(), name) +} - pathFriendlyName := CleanName(name) +func RootFsDir(name string) string { + return path.Join(SourceDir(name), "rootfs") +} - configFile := path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") +func ImageParentDir() string { + return path.Join(config.LocalStateDir, "provision/vnfs/") +} - if util.IsFile(configFile) == false { - return ret, errors.New("VNFS has not been imported: " + name) - } +func ImageFile(name string) string { + return path.Join(ImageParentDir(), name+".img.gz") +} - data, err := ioutil.ReadFile(configFile) +func ListSources() ([]string, error) { + var ret []string + + err := os.MkdirAll(SourceParentDir(), 0755) if err != nil { - return ret, errors.New("Error reading VNFS configuration file: " + name) + return ret, errors.New("Could not create VNFS source parent directory: " + SourceParentDir()) } + wwlog.Printf(wwlog.DEBUG, "Searching for VNFS Rootfs directories: %s\n", SourceParentDir()) - err = yaml.Unmarshal(data, &ret) + sources, err := ioutil.ReadDir(SourceParentDir()) if err != nil { return ret, err } - return ret, nil -} + for _, source := range sources { + wwlog.Printf(wwlog.VERBOSE, "Found VNFS source: %s\n", source.Name()) -func CleanName(source string) string { - var tmp string - - if strings.HasPrefix(source, "/") == true { - tmp = path.Base(source) - } else { - tmp = source - } - - tmp = strings.ReplaceAll(tmp, "://", ":") - tmp = strings.ReplaceAll(tmp, "/", "-") - tmp = strings.ReplaceAll(tmp, ":", ":") - - return tmp -} - -func New(source string) (VnfsObject, error) { - var ret VnfsObject - - if source == "" { - wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") - return ret, errors.New("Called vnfs.Load() without a VNFS name") - } - - if util.IsFile(ret.Config) { - return Load(source) - } - - pathFriendlyName := CleanName(source) - - if strings.HasPrefix(source, "/") == true { - ret.Source = source - ret.Name = pathFriendlyName - } else { - tmp := strings.ReplaceAll(source, "://", "-") - tmp = strings.ReplaceAll(tmp, "/", ".") - tmp = strings.ReplaceAll(tmp, ":", ".") - ret.Name = source - ret.Source = source - } - - ret.Chroot = config.VnfsChroot(pathFriendlyName) - ret.Image = config.VnfsImage(pathFriendlyName) - ret.Config = path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") - - return ret, nil -} - -func (self *VnfsObject) SaveConfig() error { - - out, err := yaml.Marshal(self) - if err != nil { - return err - } - - file, err := os.OpenFile(self.Config, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - os.Exit(1) - } - - defer file.Close() - - _, err = file.WriteString(string(out)) - if err != nil { - return err - } - - return nil -} - -func Build(name string, force bool) error { - - vnfs, err := New(name) - if err != nil { - return err - } - - wwlog.Printf(wwlog.VERBOSE, "Building VNFS: %s\n", vnfs.Name) - if strings.HasPrefix(vnfs.Source, "/") { - if strings.HasSuffix(vnfs.Source, "tar.gz") { - //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) - wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", vnfs.Name) - } else { - BuildContainerdir(vnfs, force) + if ValidName(source.Name()) == false { + continue } - } else { - BuildDocker(vnfs, force) + + if ValidSource(source.Name()) == false { + continue + } + + ret = append(ret, source.Name()) } - err = vnfs.SaveConfig() - if err != nil { - return err - } - - return nil + return ret, nil } -func (self *VnfsObject) Nameold() string { - if self.Source == "" { - return "" +func ValidSource(name string) bool { + fullPath := RootFsDir(name) + + if ValidName(name) == false { + return false } - if strings.HasPrefix(self.Source, "/") { - return path.Base(self.Source) + if util.IsDir(fullPath) == false { + wwlog.Printf(wwlog.VERBOSE, "Location is not a VNFS source directory: %s\n", name) + return false } - return self.Source -} - -func NameClean1(SourcePath string) string { - if SourcePath == "" { - return "" - } - - if strings.HasPrefix(SourcePath, "/") { - return path.Base(SourcePath) - } - uri := strings.Split(SourcePath, "://") - - return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") + if util.IsFile(path.Join(fullPath, "/sbin/init")) == false { + wwlog.Printf(wwlog.VERBOSE, "VNFS Source does not have a valid /sbin/init: %s\n", name) + return false + } + + return true } diff --git a/internal/pkg/vnfs/containerdir.go b/internal/pkg/vnfs_old/containerdir.go similarity index 100% rename from internal/pkg/vnfs/containerdir.go rename to internal/pkg/vnfs_old/containerdir.go diff --git a/internal/pkg/vnfs/docker.go b/internal/pkg/vnfs_old/docker.go similarity index 100% rename from internal/pkg/vnfs/docker.go rename to internal/pkg/vnfs_old/docker.go diff --git a/internal/pkg/vnfs/util.go b/internal/pkg/vnfs_old/util.go similarity index 100% rename from internal/pkg/vnfs/util.go rename to internal/pkg/vnfs_old/util.go diff --git a/internal/pkg/vnfs_old/vnfs.go b/internal/pkg/vnfs_old/vnfs.go new file mode 100644 index 00000000..32454fb9 --- /dev/null +++ b/internal/pkg/vnfs_old/vnfs.go @@ -0,0 +1,173 @@ +package vnfs + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/errors" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "gopkg.in/yaml.v2" + "io/ioutil" + "os" + "path" + "strings" +) + +type VnfsObject struct { + Name string + Source string + Chroot string + Image string + Config string +} + +func Load(name string) (VnfsObject, error) { + var ret VnfsObject + + if name == "" { + wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") + return ret, errors.New("Called vnfs.Load() without a VNFS name") + } + + pathFriendlyName := CleanName(name) + + configFile := path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") + + if util.IsFile(configFile) == false { + return ret, errors.New("VNFS has not been imported: " + name) + } + + data, err := ioutil.ReadFile(configFile) + if err != nil { + return ret, errors.New("Error reading VNFS configuration file: " + name) + } + + err = yaml.Unmarshal(data, &ret) + if err != nil { + return ret, err + } + + return ret, nil +} + +func CleanName(source string) string { + var tmp string + + if strings.HasPrefix(source, "/") == true { + tmp = path.Base(source) + } else { + tmp = source + } + + tmp = strings.ReplaceAll(tmp, "://", ":") + tmp = strings.ReplaceAll(tmp, "/", "-") + tmp = strings.ReplaceAll(tmp, ":", ":") + + return tmp +} + +func New(source string) (VnfsObject, error) { + var ret VnfsObject + + if source == "" { + wwlog.Printf(wwlog.DEBUG, "Called vnfs.Load() without a name, returning error\n") + return ret, errors.New("Called vnfs.Load() without a VNFS name") + } + + if util.IsFile(ret.Config) { + return Load(source) + } + + pathFriendlyName := CleanName(source) + + if strings.HasPrefix(source, "/") == true { + ret.Source = source + ret.Name = pathFriendlyName + } else { + tmp := strings.ReplaceAll(source, "://", "-") + tmp = strings.ReplaceAll(tmp, "/", ".") + tmp = strings.ReplaceAll(tmp, ":", ".") + ret.Name = source + ret.Source = source + } + + ret.Chroot = config.VnfsChroot(pathFriendlyName) + ret.Image = config.VnfsImage(pathFriendlyName) + ret.Config = path.Join(config.VnfsImageDir(pathFriendlyName), "config.yaml") + + return ret, nil +} + +func (self *VnfsObject) SaveConfig() error { + + out, err := yaml.Marshal(self) + if err != nil { + return err + } + + file, err := os.OpenFile(self.Config, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + + defer file.Close() + + _, err = file.WriteString(string(out)) + if err != nil { + return err + } + + return nil +} + +func Build(name string, force bool) error { + + vnfs, err := New(name) + if err != nil { + return err + } + + wwlog.Printf(wwlog.VERBOSE, "Building VNFS: %s\n", vnfs.Name) + if strings.HasPrefix(vnfs.Source, "/") { + if strings.HasSuffix(vnfs.Source, "tar.gz") { + //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) + wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", vnfs.Name) + } else { + BuildContainerdir(vnfs, force) + } + } else { + BuildDocker(vnfs, force) + } + + err = vnfs.SaveConfig() + if err != nil { + return err + } + + return nil +} + +func (self *VnfsObject) Nameold() string { + if self.Source == "" { + return "" + } + + if strings.HasPrefix(self.Source, "/") { + return path.Base(self.Source) + } + + return self.Source +} + +func NameClean1(SourcePath string) string { + if SourcePath == "" { + return "" + } + + if strings.HasPrefix(SourcePath, "/") { + return path.Base(SourcePath) + } + uri := strings.Split(SourcePath, "://") + + return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") +}