Major refactor for Cobra and new pkg/app structure for wwctl

This commit is contained in:
Gregory Kurtzer
2020-11-10 21:28:39 -08:00
parent 0c369270ef
commit 189618a111
24 changed files with 658 additions and 130 deletions

View File

@@ -0,0 +1,79 @@
package build
import (
"github.com/hpcng/warewulf/internal/app/wwctl/build/kernel"
runtime_overlay "github.com/hpcng/warewulf/internal/app/wwctl/build/runtime-overlay"
system_overlay "github.com/hpcng/warewulf/internal/app/wwctl/build/system-overlay"
"github.com/hpcng/warewulf/internal/app/wwctl/build/vnfs"
"github.com/hpcng/warewulf/internal/pkg/assets"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"github.com/spf13/cobra"
"os"
)
func CobraRunE(cmd *cobra.Command, args []string) error {
var nodeList []assets.NodeInfo
if buildAll == true {
wwlog.Printf(wwlog.VERBOSE, "Building all components\n")
buildVnfs = true
buildKernel = true
buildSystemOverlay = true;
buildRuntimeOverlay = true;
}
if len(args) >= 1 {
nodeList, _ = assets.SearchByName(args[0])
} else {
nodeList, _ = assets.FindAllNodes()
}
if len(nodeList) == 0 {
wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0])
os.Exit(255)
} else {
wwlog.Printf(wwlog.VERBOSE, "Found matching nodes for build: %d\n", len(nodeList))
}
if buildVnfs == true {
// wwlog.Printf(wwlog.INFO, "===============================================================================\n")
err := vnfs.Build(nodeList, buildForce)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(255)
}
}
if buildKernel == true {
// wwlog.Printf(wwlog.INFO, "===============================================================================\n")
err := kernel.Build(nodeList, buildForce)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(255)
}
}
if buildSystemOverlay == true {
// wwlog.Printf(wwlog.INFO, "===============================================================================\n")
err := system_overlay.Build(nodeList, buildForce)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(255)
}
}
if buildRuntimeOverlay == true {
// wwlog.Printf(wwlog.INFO, "===============================================================================\n")
err := runtime_overlay.Build(nodeList, buildForce)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(255)
}
}
return nil
}

View File

@@ -0,0 +1,74 @@
package kernel
import (
"fmt"
"github.com/hpcng/warewulf/internal/pkg/assets"
"github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"os"
"os/exec"
"path"
)
const kernelProvisionPath = "/provision/kernel/"
func Build(nodeList []assets.NodeInfo, force bool) error {
set := make(map[string]int)
wwlog.Printf(wwlog.INFO, "Importing Kernels:\n")
wwlog.SetIndent(4)
for _, node := range nodeList {
if node.KernelVersion != "" {
set[node.KernelVersion] ++
wwlog.Printf(wwlog.DEBUG, "Node '%s' has KernelVersion '%s'\n", node.Fqdn, node.KernelVersion)
}
}
for kernelVersion := range set {
kernelImage := "/boot/vmlinuz-"+kernelVersion
kernelDrivers := "/lib/modules/"+kernelVersion
kernelDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "vmlinuz-"+kernelVersion)
driversDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "kmods-"+kernelVersion+".img")
// Create the kernel destination path just in case it doesn't exist
os.MkdirAll(path.Join(config.LocalStateDir, kernelProvisionPath), 0755)
if _, err := os.Stat(kernelImage); err == nil {
if util.PathIsNewer(kernelImage, kernelDestination) && force == false {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, kernel is current\n", "vmlinuz-"+kernelVersion)
} else {
err := util.CopyFile(kernelImage, kernelDestination)
if err != nil {
wwlog.Printf(wwlog.ERROR, "Failed copying kernel image: %s\n", err)
continue
}
wwlog.Printf(wwlog.INFO, "%-35s: Done\n", "vmlinuz-"+kernelVersion)
}
}
if _, err := os.Stat(kernelDrivers); err == nil {
if util.PathIsNewer(kernelDrivers, driversDestination) && force == false {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, kernel Drivers are current\n", "kmods-"+kernelVersion+".img")
} else {
cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelDrivers, driversDestination)
err := exec.Command("/bin/sh", "-c", cmd).Run()
if err != nil {
wwlog.Printf(wwlog.ERROR, "Could not generate kernel driver overlay: %s\n", err)
continue
}
wwlog.Printf(wwlog.INFO, "%-35s: Done\n", "kmods-"+kernelVersion+".img")
}
}
}
wwlog.SetIndent(0)
return nil
}

View File

@@ -1,7 +1,6 @@
package build
import (
"fmt"
"github.com/spf13/cobra"
)
@@ -9,16 +8,26 @@ var (
buildCmd = &cobra.Command{
Use: "build",
Short: "Warewulf build subcommand",
Long: "Warewulf build is used to build VNFS, kernel, and overlay objects for provisioning.",
RunE: buildRunE,
Long: "Warewulf build is used to build VNFS, kernel, and system-overlay objects for\n" +
"provisioning. The default usage will be to build and/or update anything\n" +
"that seems to be needed.",
RunE: CobraRunE,
}
option string
buildVnfs bool
buildKernel bool
buildRuntimeOverlay bool
buildSystemOverlay bool
buildAll bool
buildForce bool
)
func init() {
//buildCmd.PersistentFlags().StringVar(&flagHost, FlagHost, "localhost:7331", "Address of the Fuzzball Host.")
buildCmd.PersistentFlags().StringVarP(&option, "testopt", "t", "default", "This is a test option")
buildCmd.PersistentFlags().BoolVarP(&buildVnfs, "vnfs", "V", false, "Build and/or update VNFS images.")
buildCmd.PersistentFlags().BoolVarP(&buildKernel, "kernel", "K", false, "Build and/or update Kernel images.")
buildCmd.PersistentFlags().BoolVarP(&buildRuntimeOverlay, "runtime", "R", false, "Build and/or update runtime overlays")
buildCmd.PersistentFlags().BoolVarP(&buildSystemOverlay, "system", "S", false, "Build and/or update system overlays")
buildCmd.PersistentFlags().BoolVarP(&buildAll, "all", "A", false, "Build and/or update all components")
buildCmd.PersistentFlags().BoolVarP(&buildForce, "force", "f", false, "Force build even if nothing has been updated.")
}
@@ -26,10 +35,3 @@ func init() {
func GetCommand() *cobra.Command {
return buildCmd
}
func buildRunE(cmd *cobra.Command, args []string) error {
fmt.Printf("Hello World. Got Option: %s, %s\n", option, args)
return nil
}

View File

@@ -0,0 +1,133 @@
package runtime_overlay
import (
"fmt"
"github.com/hpcng/warewulf/internal/pkg/assets"
"github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
"text/template"
)
func Build(nodeList []assets.NodeInfo, force bool) error {
wwlog.Printf(wwlog.INFO, "Building Runtime Overlays:\n")
wwlog.SetIndent(4)
for _, node := range nodeList {
if node.RuntimeOverlay != "" {
OverlayDir := path.Join(config.LocalStateDir, "/overlays/runtime/", node.RuntimeOverlay)
OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/runtime/", node.Fqdn+".img")
wwlog.Printf(wwlog.VERBOSE, "Building Runtime Overlay for: %s\n", node.Fqdn)
tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-runtime-overlay-")
if err != nil {
return err
}
if _, err := os.Stat(OverlayDir); err != nil {
return err
}
err = os.MkdirAll(path.Dir(OverlayFile), 0755)
if err != nil {
return err
}
if force == false {
wwlog.Printf(wwlog.DEBUG, "Checking if overlay is required\n")
}
if util.PathIsNewer(OverlayDir, OverlayFile) {
if force == false {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, overlay is current\n", node.Fqdn)
continue
}
}
wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlayDir)
err = os.Chdir(OverlayDir)
if err != nil {
wwlog.Printf(wwlog.ERROR, "Could not chdir() to OverlayDir: %s\n", OverlayDir)
continue
}
wwlog.Printf(wwlog.DEBUG, "Walking the file system: %s\n", OverlayDir)
err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location)
err := os.MkdirAll(path.Join(tmpDir, location), info.Mode())
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
} else if filepath.Ext(location) == ".ww" {
wwlog.Printf(wwlog.DEBUG, "Found template file: %s\n", location)
destFile := strings.TrimSuffix(location, ".ww")
tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*")
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
w, err := os.OpenFile(path.Join(tmpDir, destFile), os.O_RDWR|os.O_CREATE, info.Mode())
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
defer w.Close()
err = tmpl.Execute(w, node)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
} else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b == true {
wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location)
} else {
wwlog.Printf(wwlog.DEBUG, "Found file: %s\n", location)
err := util.CopyFile(path.Join(OverlayDir, location), path.Join(tmpDir, location))
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
}
return nil
})
wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", node.Fqdn)
cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, OverlayFile)
wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd)
err = exec.Command("/bin/sh", "-c", cmd).Run()
if err != nil {
wwlog.Printf(wwlog.ERROR, "Could not generate runtime image overlay: %s\n", err)
continue
}
wwlog.Printf(wwlog.INFO, "%-3s: Done\n", node.Fqdn)
wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir)
os.RemoveAll(tmpDir)
}
}
wwlog.SetIndent(0)
return nil
}

View File

@@ -0,0 +1,133 @@
package system_overlay
import (
"fmt"
"github.com/hpcng/warewulf/internal/pkg/assets"
"github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
"text/template"
)
func Build(nodeList []assets.NodeInfo, force bool) error {
wwlog.Printf(wwlog.INFO, "Building System Overlays:\n")
wwlog.SetIndent(4)
for _, node := range nodeList {
if node.SystemOverlay != "" {
OverlayDir := path.Join(config.LocalStateDir, "/overlays/system/", node.SystemOverlay)
OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/system/", node.Fqdn+".img")
wwlog.Printf(wwlog.VERBOSE, "Building System Overlay for: %s\n", node.Fqdn)
tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-system-overlay-")
if err != nil {
return err
}
if _, err := os.Stat(OverlayDir); err != nil {
return err
}
err = os.MkdirAll(path.Dir(OverlayFile), 0755)
if err != nil {
return err
}
if force == false {
wwlog.Printf(wwlog.DEBUG, "Checking if overlay is required\n")
}
if util.PathIsNewer(OverlayDir, OverlayFile) {
if force == false {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, overlay is current\n", node.Fqdn)
continue
}
}
wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlayDir)
err = os.Chdir(OverlayDir)
if err != nil {
wwlog.Printf(wwlog.ERROR, "Could not chdir() to OverlayDir: %s\n", OverlayDir)
continue
}
wwlog.Printf(wwlog.DEBUG, "Walking the file system: %s\n", OverlayDir)
err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location)
err := os.MkdirAll(path.Join(tmpDir, location), info.Mode())
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
} else if filepath.Ext(location) == ".ww" {
wwlog.Printf(wwlog.DEBUG, "Found template file: %s\n", location)
destFile := strings.TrimSuffix(location, ".ww")
tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*")
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
w, err := os.OpenFile(path.Join(tmpDir, destFile), os.O_RDWR|os.O_CREATE, info.Mode())
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
defer w.Close()
err = tmpl.Execute(w, node)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
} else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b == true {
wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location)
} else {
wwlog.Printf(wwlog.DEBUG, "Found file: %s\n", location)
err := util.CopyFile(path.Join(OverlayDir, location), path.Join(tmpDir, location))
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
return err
}
}
return nil
})
wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", node.Fqdn)
cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, OverlayFile)
wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd)
err = exec.Command("/bin/sh", "-c", cmd).Run()
if err != nil {
wwlog.Printf(wwlog.ERROR, "Could not generate system image overlay: %s\n", err)
continue
}
wwlog.Printf(wwlog.INFO, "%-35s: Done\n", node.Fqdn)
wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir)
os.RemoveAll(tmpDir)
}
}
wwlog.SetIndent(0)
return nil
}

View File

@@ -0,0 +1,50 @@
package vnfs
import (
"fmt"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/vnfs"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"os"
"path"
)
func BuildContainerdir(v vnfs.VnfsObject) {
if _, err := os.Stat(v.Source()); err != nil {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping (bad path)\n", v.Name())
return
}
wwlog.Printf(wwlog.DEBUG, "Checking if there have been any updates to the VNFS directory\n")
if util.PathIsNewer(v.Source(), v.Image()) {
if buildForce == false {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name())
return
}
}
wwlog.Printf(wwlog.DEBUG, "Making the directory: %s\n", path.Dir(v.Image()))
err := os.MkdirAll(path.Dir(v.Image()), 0755)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
return
}
wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", v.Source(), v.Image())
err = buildVnfs(v.Source(), v.Image())
if err != nil {
fmt.Printf("ERROR: %s\n", err)
os.Exit(1)
}
wwlog.Printf(wwlog.DEBUG, "Building links for Warewulf access to chroot\n")
err = buildLinks(v, v.Source())
if err != nil {
fmt.Printf("ERROR: %s\n", err)
os.Exit(1)
}
wwlog.Printf(wwlog.INFO, "%-35s: Done\n", v.Name())
}

View File

@@ -0,0 +1,89 @@
package vnfs
import (
"context"
"fmt"
"github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/oci"
"github.com/hpcng/warewulf/internal/pkg/vnfs"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"os"
"path"
)
const (
OciCacheDir = config.LocalStateDir + "/oci"
VnfsHashDir = config.LocalStateDir + "/oci/vnfs/hash/"
)
func BuildDocker(v vnfs.VnfsObject) {
wwlog.Printf(wwlog.VERBOSE, "Building OCI Container: %s\n", v.Source())
c, err := oci.NewCache(oci.OptSetCachePath(OciCacheDir))
if err != nil {
fmt.Printf("ERROR: %s\n", err)
os.Exit(1)
}
wwlog.Printf(wwlog.VERBOSE, "Downloading OCI container layers\n")
sourcePath, err := c.Pull(context.Background(), v.Source(), nil)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
os.Exit(1)
}
hashDestination := VnfsHashDir + path.Base(sourcePath)
name, err := os.Readlink(v.Image())
if err == nil {
if name == hashDestination {
wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name())
return
}
}
err = os.MkdirAll(VnfsHashDir, 0755)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
return
}
err = os.MkdirAll(path.Dir(v.Image()), 0755)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
return
}
err = os.MkdirAll(path.Dir(v.Root()), 0755)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
return
}
wwlog.Printf(wwlog.VERBOSE, "Building bootable VNFS image\n")
err = buildVnfs(sourcePath, hashDestination)
if err != nil {
fmt.Printf("ERROR: %s\n", err)
os.Exit(1)
}
wwlog.Printf(wwlog.VERBOSE, "Finalizing Build\n")
_ = os.Remove(v.Image() + "-link")
err = os.Symlink(hashDestination, v.Image()+"-link")
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(1)
}
err = os.Rename(v.Image()+"-link", v.Image())
err = buildLinks(v, sourcePath)
if err != nil {
wwlog.Printf(wwlog.ERROR, "%s\n", err)
os.Exit(1)
}
wwlog.Printf(wwlog.INFO, "%-35s: Done\n", v.Name())
return
}

View File

@@ -0,0 +1,41 @@
package vnfs
import (
"fmt"
"github.com/hpcng/warewulf/internal/pkg/vnfs"
"os"
"os/exec"
"path"
)
func buildVnfs(source string, dest string) error {
cmd := fmt.Sprintf("cd %s; find . | cpio --quiet -o -H newc | gzip -c > \"%s\"", source, dest)
err := exec.Command("/bin/sh", "-c", cmd).Run()
return err
}
func buildLinks(v vnfs.VnfsObject, source string) error {
// Just incase the temporary link location is present, remove it if we can
_ = os.Remove(v.Root() + "-link")
// Just incase the directory doesn't exist, make it
_ = os.MkdirAll(path.Dir(v.Root()), 0755)
// Link to a temporary location so we can atomically move the link into place
err := os.Symlink(source, v.Root()+"-link")
if err != nil {
return err
}
// Atomically move the link into place
err = os.Rename(v.Root()+"-link", v.Root())
if err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,51 @@
package vnfs
import (
"github.com/hpcng/warewulf/internal/pkg/assets"
"github.com/hpcng/warewulf/internal/pkg/vnfs"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"strings"
)
var buildForce bool
func Build(nodeList []assets.NodeInfo, force bool) (error) {
set := make(map[string]int)
wwlog.Printf(wwlog.INFO, "Importing VNFS Images:\n")
wwlog.SetIndent(4)
buildForce = force
for _, node := range nodeList {
if node.Vnfs != "" {
set[node.Vnfs] ++
wwlog.Printf(wwlog.DEBUG, "Node '%s' has VNFS '%s'\n", node.Fqdn, node.Vnfs)
}
}
for uri := range set {
v := vnfs.New(uri)
wwlog.Printf(wwlog.VERBOSE, "VNFS found: %s (nodes: %d)\n", uri, set[uri])
if strings.HasPrefix(uri, "docker://") {
BuildDocker(v)
} else if strings.HasPrefix(uri, "docker-daemon://") {
//wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service: %s\n", uri)
wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service is not supported yet: %s\n", uri)
} else if strings.HasPrefix(uri, "/") {
if strings.HasSuffix(uri, "tar.gz") {
//wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri)
wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", uri)
} else {
BuildContainerdir(v)
}
}
}
wwlog.SetIndent(0)
return nil
}

View File

@@ -9,16 +9,14 @@ import (
var (
rootCmd = &cobra.Command{
Use: "wwctl",
Short: "Warewulf CTL",
Long: "Fuzzball CLI is an application for interacting with a Fuzzball Service.",
Short: "Warewulf Control",
Long: "Control interface to the Cluster Warewulf Provisioning System.",
PersistentPreRunE: rootPersistentPreRunE,
}
verboseArg bool
debugArg bool
)
func init() {
rootCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.")
rootCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.")