From f3ec8ee923314f1fd49e780028be5a855b7c235a Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Mon, 9 Nov 2020 17:24:35 -0800 Subject: [PATCH 01/15] Lots of cleanups, added vnfs package and utilizing config package for constants --- cmd/warewulfd/ipxe.go | 4 +- cmd/warewulfd/kernel.go | 5 ++- cmd/warewulfd/kmods.go | 5 ++- cmd/warewulfd/runtime.go | 4 +- cmd/warewulfd/system.go | 5 ++- cmd/warewulfd/vnfs.go | 11 +++-- cmd/warewulfd/warewulfd.go | 14 +++--- cmd/wwbuild/overlay-runtime.go | 59 +++++++++++++------------- cmd/wwbuild/overlay-system.go | 9 ++-- cmd/wwbuild/vnfs-local.go | 14 ++++-- cmd/wwbuild/vnfs-oci.go | 43 +++++++++++++------ cmd/wwbuild/vnfs.go | 22 ++++++++++ cmd/wwbuild/wwbuild.go | 10 +++-- etc/nodes.conf | 2 +- internal/pkg/assets/assets.go | 5 ++- internal/pkg/config/config.go | 24 ++++++++--- internal/pkg/oci/puller.go | 4 +- internal/pkg/vnfs/vnfs.go | 51 ++++++++++++++++++++++ overlays/runtime/default/etc/group.in | 4 +- overlays/runtime/default/etc/passwd.in | 4 +- 20 files changed, 206 insertions(+), 93 deletions(-) create mode 100644 internal/pkg/vnfs/vnfs.go diff --git a/cmd/warewulfd/ipxe.go b/cmd/warewulfd/ipxe.go index 534f6baf..e735df80 100644 --- a/cmd/warewulfd/ipxe.go +++ b/cmd/warewulfd/ipxe.go @@ -12,7 +12,7 @@ import ( "strings" ) -func ipxe(w http.ResponseWriter, req *http.Request) { +func ipxeSend(w http.ResponseWriter, req *http.Request) { url := strings.Split(req.URL.Path, "/") if url[2] == "" { @@ -58,12 +58,10 @@ func ipxe(w http.ResponseWriter, req *http.Request) { // TODO: Add KernelArgs to nodes.conf //newLine = strings.ReplaceAll(newLine, "@KERNELARGS@", node.KernelArgs) - fmt.Fprintln(w, newLine) } log.Printf("SEND: %15s: %s\n", node.Fqdn, ipxeTemplate) - } else { log.Printf("ERROR: iPXE request from unknown Node (hwaddr=%s)\n", url[2]) } diff --git a/cmd/warewulfd/kernel.go b/cmd/warewulfd/kernel.go index 10a5477b..db406f6c 100644 --- a/cmd/warewulfd/kernel.go +++ b/cmd/warewulfd/kernel.go @@ -2,11 +2,12 @@ package main import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" "log" "net/http" ) -func kernel(w http.ResponseWriter, req *http.Request) { +func kernelSend(w http.ResponseWriter, req *http.Request) { node, err := getSanity(req) if err != nil { @@ -16,7 +17,7 @@ func kernel(w http.ResponseWriter, req *http.Request) { } if node.KernelVersion != "" { - fileName := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", LocalStateDir, node.KernelVersion) + fileName := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", config.LocalStateDir, node.KernelVersion) err := sendFile(w, fileName, node.Fqdn) if err != nil { diff --git a/cmd/warewulfd/kmods.go b/cmd/warewulfd/kmods.go index 0a090811..7af8de57 100644 --- a/cmd/warewulfd/kmods.go +++ b/cmd/warewulfd/kmods.go @@ -2,11 +2,12 @@ package main import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" "log" "net/http" ) -func kmods(w http.ResponseWriter, req *http.Request) { +func kmodsSend(w http.ResponseWriter, req *http.Request) { node, err := getSanity(req) if err != nil { @@ -15,7 +16,7 @@ func kmods(w http.ResponseWriter, req *http.Request) { return } if node.KernelVersion != "" { - fileName := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", LocalStateDir, node.KernelVersion) + fileName := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", config.LocalStateDir, node.KernelVersion) err := sendFile(w, fileName, node.Fqdn) if err != nil { diff --git a/cmd/warewulfd/runtime.go b/cmd/warewulfd/runtime.go index 0cc793f7..7ee88283 100644 --- a/cmd/warewulfd/runtime.go +++ b/cmd/warewulfd/runtime.go @@ -10,7 +10,7 @@ import ( "strings" ) -func runtimeOverlay(w http.ResponseWriter, req *http.Request) { +func runtimeOverlaySend(w http.ResponseWriter, req *http.Request) { remote := strings.Split(req.RemoteAddr, ":") port, err := strconv.Atoi(remote[1]) if err != nil { @@ -49,7 +49,7 @@ func runtimeOverlay(w http.ResponseWriter, req *http.Request) { } if node.RuntimeOverlay != "" { - fileName := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", LocalStateDir, node.Fqdn) + fileName := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", config.LocalStateDir, node.Fqdn) err := sendFile(w, fileName, node.Fqdn) if err != nil { diff --git a/cmd/warewulfd/system.go b/cmd/warewulfd/system.go index 3f3f6ea9..7efe428b 100644 --- a/cmd/warewulfd/system.go +++ b/cmd/warewulfd/system.go @@ -2,11 +2,12 @@ package main import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" "log" "net/http" ) -func systemOverlay(w http.ResponseWriter, req *http.Request) { +func systemOverlaySend(w http.ResponseWriter, req *http.Request) { node, err := getSanity(req) if err != nil { @@ -16,7 +17,7 @@ func systemOverlay(w http.ResponseWriter, req *http.Request) { } if node.SystemOverlay != "" { - fileName := fmt.Sprintf("%s/provision/overlays/system/%s.img", LocalStateDir, node.Fqdn) + fileName := fmt.Sprintf("%s/provision/overlays/system/%s.img", config.LocalStateDir, node.Fqdn) err := sendFile(w, fileName, node.Fqdn) if err != nil { diff --git a/cmd/warewulfd/vnfs.go b/cmd/warewulfd/vnfs.go index b7747a2a..675a3ffc 100644 --- a/cmd/warewulfd/vnfs.go +++ b/cmd/warewulfd/vnfs.go @@ -1,13 +1,12 @@ package main import ( - "fmt" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "log" "net/http" - "path" ) -func vnfs(w http.ResponseWriter, req *http.Request) { +func vnfsSend(w http.ResponseWriter, req *http.Request) { node, err := getSanity(req) if err != nil { @@ -17,13 +16,13 @@ func vnfs(w http.ResponseWriter, req *http.Request) { } if node.Vnfs != "" { - fileName := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(node.Vnfs)) + v := vnfs.New(node.Vnfs) - err := sendFile(w, fileName, node.Fqdn) + err := sendFile(w, v.Image(), node.Fqdn) if err != nil { log.Printf("ERROR: %s\n", err) } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) + log.Printf("SEND: %15s: %s\n", node.Fqdn, v.Image()) } } else { w.WriteHeader(503) diff --git a/cmd/warewulfd/warewulfd.go b/cmd/warewulfd/warewulfd.go index 912c1516..1cf34ed0 100644 --- a/cmd/warewulfd/warewulfd.go +++ b/cmd/warewulfd/warewulfd.go @@ -15,7 +15,7 @@ import ( // TODO: https://github.com/danderson/netboot/blob/master/pixiecore/dhcp.go // TODO: https://github.com/pin/tftp -const LocalStateDir = "/var/warewulf" +//const LocalStateDir = "/var/warewulf" func getSanity(req *http.Request) (assets.NodeInfo, error) { url := strings.Split(req.URL.Path, "/") @@ -62,12 +62,12 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) error { func main() { - http.HandleFunc("/ipxe/", ipxe) - http.HandleFunc("/kernel/", kernel) - http.HandleFunc("/kmods/", kmods) - http.HandleFunc("/vnfs/", vnfs) - http.HandleFunc("/overlay-system/", systemOverlay) - http.HandleFunc("/overlay-runtime", runtimeOverlay) + http.HandleFunc("/ipxe/", ipxeSend) + http.HandleFunc("/kernel/", kernelSend) + http.HandleFunc("/kmods/", kmodsSend) + http.HandleFunc("/vnfs/", vnfsSend) + http.HandleFunc("/overlay-system/", systemOverlaySend) + http.HandleFunc("/overlay-runtime", runtimeOverlaySend) http.ListenAndServe(":9873", nil) } diff --git a/cmd/wwbuild/overlay-runtime.go b/cmd/wwbuild/overlay-runtime.go index a1ca74b7..6e077c87 100644 --- a/cmd/wwbuild/overlay-runtime.go +++ b/cmd/wwbuild/overlay-runtime.go @@ -3,6 +3,7 @@ package main import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/util" "os" "os/exec" @@ -13,22 +14,22 @@ import ( func overlayRuntime(node assets.NodeInfo, replace map[string]string, wg *sync.WaitGroup) { defer wg.Done() - OverlayDir := fmt.Sprintf("%s/overlays/runtime/%s", LocalStateDir, node.RuntimeOverlay) - OverlayFile := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", LocalStateDir, node.Fqdn) -/* - destModTime := time.Time{} - destMod, err := os.Stat(OverlayFile) - if err == nil { - destModTime = destMod.ModTime() - } - configMod, err := os.Stat("/etc/warewulf/nodes.conf") - if err != nil { - fmt.Printf("ERROR: could not find node file: /etc/warewulf/nodes.conf") - os.Exit(1) - } - configModTime := configMod.ModTime() - sourceModTime, _ := util.DirModTime(OverlayDir) -*/ + OverlayDir := fmt.Sprintf("%s/overlays/runtime/%s", config.LocalStateDir, node.RuntimeOverlay) + OverlayFile := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", config.LocalStateDir, node.Fqdn) + /* + destModTime := time.Time{} + destMod, err := os.Stat(OverlayFile) + if err == nil { + destModTime = destMod.ModTime() + } + configMod, err := os.Stat("/etc/warewulf/nodes.conf") + if err != nil { + fmt.Printf("ERROR: could not find node file: /etc/warewulf/nodes.conf") + os.Exit(1) + } + configModTime := configMod.ModTime() + sourceModTime, _ := util.DirModTime(OverlayDir) + */ err := os.MkdirAll(path.Dir(OverlayFile), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) @@ -39,20 +40,20 @@ func overlayRuntime(node assets.NodeInfo, replace map[string]string, wg *sync.Wa fmt.Printf("ERROR: %s\n", err) return } -// if sourceModTime.After(destModTime) || configModTime.After(destModTime) { - fmt.Printf("RUNTIME: %s\n", node.Fqdn) + // if sourceModTime.After(destModTime) || configModTime.After(destModTime) { + fmt.Printf("RUNTIME: %s\n", node.Fqdn) - overlayDest := "/tmp/.overlay-" + util.RandomString(16) - BuildOverlayDir(OverlayDir, overlayDest, replace) + overlayDest := "/tmp/.overlay-" + util.RandomString(16) + BuildOverlayDir(OverlayDir, overlayDest, replace) - cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) - err = exec.Command("/bin/sh", "-c", cmd).Run() - if err != nil { - fmt.Printf("%s", err) - } + cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + fmt.Printf("%s", err) + } - os.RemoveAll(overlayDest) -// } else { -// fmt.Printf("RUNTIME: %s (skipped no changes)\n", node.Fqdn) -// } + os.RemoveAll(overlayDest) + // } else { + // fmt.Printf("RUNTIME: %s (skipped no changes)\n", node.Fqdn) + // } } diff --git a/cmd/wwbuild/overlay-system.go b/cmd/wwbuild/overlay-system.go index eaba6315..0cfaca23 100644 --- a/cmd/wwbuild/overlay-system.go +++ b/cmd/wwbuild/overlay-system.go @@ -3,6 +3,7 @@ package main import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/util" "os" "os/exec" @@ -14,17 +15,17 @@ import ( func overlaySystem(node assets.NodeInfo, replace map[string]string, wg *sync.WaitGroup) { defer wg.Done() - OverlayDir := fmt.Sprintf("%s/overlays/system/%s", LocalStateDir, node.SystemOverlay) - OverlayFile := fmt.Sprintf("%s/provision/overlays/system/%s.img", LocalStateDir, node.Fqdn) + OverlayDir := fmt.Sprintf("%s/overlays/system/%s", config.LocalStateDir, node.SystemOverlay) + OverlayFile := fmt.Sprintf("%s/provision/overlays/system/%s.img", config.LocalStateDir, node.Fqdn) destModTime := time.Time{} destMod, err := os.Stat(OverlayFile) if err == nil { destModTime = destMod.ModTime() } - configMod, err := os.Stat("/etc/warewulf/nodes.conf") + configMod, err := os.Stat(config.SysConfDir + "nodes.conf") if err != nil { - fmt.Printf("ERROR: could not find node file: /etc/warewulf/nodes.conf\n") + fmt.Printf("ERROR: could not find node file: nodes.conf\n", config.SysConfDir) os.Exit(1) } configModTime := configMod.ModTime() diff --git a/cmd/wwbuild/vnfs-local.go b/cmd/wwbuild/vnfs-local.go index 4a0fdb0c..91e5babb 100644 --- a/cmd/wwbuild/vnfs-local.go +++ b/cmd/wwbuild/vnfs-local.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "os" "path" "sync" @@ -9,22 +10,29 @@ import ( func vnfsLocalBuild(vnfsPath string, wg *sync.WaitGroup) { defer wg.Done() + v := vnfs.New(vnfsPath) if _, err := os.Stat(vnfsPath); err == nil { // TODO: Build VNFS to temporary file and move to real location when complete atomically // TODO: Check time stamps of sourcedir and build file to see if we need to rebuild or skip - vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(vnfsPath)) + //vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(vnfsPath)) fmt.Printf("Building local Container: %s\n", vnfsPath) - err := os.MkdirAll(path.Dir(vnfsDestination), 0755) + err := os.MkdirAll(path.Dir(v.Image()), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - err = buildVnfs(vnfsPath, vnfsDestination) + err = buildVnfs(vnfsPath, v.Image()) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + os.Exit(1) + } + + err = buildLinks(v, vnfsPath) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) diff --git a/cmd/wwbuild/vnfs-oci.go b/cmd/wwbuild/vnfs-oci.go index 7d15555f..399fbc0c 100644 --- a/cmd/wwbuild/vnfs-oci.go +++ b/cmd/wwbuild/vnfs-oci.go @@ -3,75 +3,90 @@ package main import ( "context" "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/oci" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "log" "os" "path" "sync" ) +const ( + OciCacheDir = config.LocalStateDir + "/oci" + VnfsHashDir = config.LocalStateDir + "/oci/vnfs/hash/" +) + func vnfsOciBuild(OciPath string, wg *sync.WaitGroup) { - vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(OciPath)) + v := vnfs.New(OciPath) + + // vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(OciPath)) defer wg.Done() log.Printf("Building OCI Container: %s\n", OciPath) - c, err := oci.NewCache(oci.OptSetCachePath("/var/warewulf/oci")) + + c, err := oci.NewCache(oci.OptSetCachePath(OciCacheDir)) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } + log.Printf("Downloading OCI container layers\n") sourcePath, err := c.Pull(context.Background(), OciPath, nil) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } - ociDestination := fmt.Sprintf("%s/oci/vnfs/hash/%s", LocalStateDir, path.Base(sourcePath)) + hashDestination := VnfsHashDir + path.Base(sourcePath) - name, err := os.Readlink(vnfsDestination) + name, err := os.Readlink(v.Image()) if err == nil { - if name == ociDestination { - log.Printf("Container already built, no changes from upstream\n") + if name == hashDestination { + log.Printf("Container already built, no update available\n") return } } - err = os.MkdirAll(path.Dir(ociDestination), 0755) + err = os.MkdirAll(VnfsHashDir, 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - err = os.MkdirAll(path.Dir(vnfsDestination), 0755) + err = os.MkdirAll(path.Dir(v.Image()), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - err = os.MkdirAll(LocalStateDir+"/oci/vnfs/name", 0755) + err = os.MkdirAll(path.Dir(v.Root()), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - err = buildVnfs(sourcePath, ociDestination) + log.Printf("Building bootable VNFS image\n") + + err = buildVnfs(sourcePath, hashDestination) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } - err = os.Symlink(ociDestination, vnfsDestination+"-link") + log.Printf("Finalizing Build\n") + + _ = os.Remove(v.Image() + "-link") + err = os.Symlink(hashDestination, v.Image()+"-link") if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } + err = os.Rename(v.Image()+"-link", v.Image()) - err = os.Symlink(sourcePath, LocalStateDir+"/oci/vnfs/name/"+path.Base(OciPath)) + err = buildLinks(v, sourcePath) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } - err = os.Rename(vnfsDestination+"-link", vnfsDestination) - log.Printf("Completed building VNFS: %s\n", path.Base(OciPath)) return diff --git a/cmd/wwbuild/vnfs.go b/cmd/wwbuild/vnfs.go index d113b0d2..f9ab7f67 100644 --- a/cmd/wwbuild/vnfs.go +++ b/cmd/wwbuild/vnfs.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "os" "os/exec" ) @@ -12,3 +14,23 @@ func buildVnfs(source string, dest string) error { return err } + +func buildLinks(v vnfs.VnfsObject, source string) error { + + // Just incase the temporary link location is present, remove it if we can + _ = os.Remove(v.Root() + "-link") + + // Link to a temporary location so we can atomically move the link into place + err := os.Symlink(source, v.Root()+"-link") + if err != nil { + return err + } + + // Atomically move the link into place + err = os.Rename(v.Root()+"-link", v.Root()) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/wwbuild/wwbuild.go b/cmd/wwbuild/wwbuild.go index 533f1db8..8daa7e4f 100644 --- a/cmd/wwbuild/wwbuild.go +++ b/cmd/wwbuild/wwbuild.go @@ -3,6 +3,8 @@ package main import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "os" "os/exec" "path" @@ -11,8 +13,6 @@ import ( "time" ) -const LocalStateDir = "/var/warewulf" - func main() { if len(os.Args) < 2 { @@ -80,8 +80,8 @@ func main() { kernelSource := fmt.Sprintf("/boot/vmlinuz-%s", kernelVers) // TODO: Check time stamps of source and dests to see if we need to rebuild or skip if _, err := os.Stat(kernelSource); err == nil { - kernelDestination := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", LocalStateDir, kernelVers) - kmodsDestination := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", LocalStateDir, kernelVers) + kernelDestination := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", config.LocalStateDir, kernelVers) + kmodsDestination := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", config.LocalStateDir, kernelVers) err := os.MkdirAll(path.Dir(kernelDestination), 0755) if err != nil { @@ -123,10 +123,12 @@ func main() { } for _, node := range nodeList { + v := vnfs.New(node.Vnfs) replace := make(map[string]string) replace["HOSTNAME"] = node.HostName replace["FQDN"] = node.Fqdn replace["VNFS"] = node.Vnfs + replace["VNFSDIR"] = v.Root() replace["KERNELVERSION"] = node.KernelVersion replace["GROUPNAME"] = node.GroupName replace["DOMAIN"] = node.DomainName diff --git a/etc/nodes.conf b/etc/nodes.conf index 6affbecc..8ca76f27 100644 --- a/etc/nodes.conf +++ b/etc/nodes.conf @@ -18,7 +18,7 @@ nodegroups: gateway: 192.168.1.1 group_2: comment: This is the group 2 - vnfs: /var/chroots/centos-7 + vnfs: docker://warewulf/centos-7 system overlay: default runtime overlay: default domain suffix: group2 diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 190b485a..eb08c4a3 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -64,6 +64,7 @@ type NodeInfo struct { DomainName string Fqdn string Vnfs string + VnfsDir string Ipxe string SystemOverlay string RuntimeOverlay string @@ -139,7 +140,9 @@ func FindAllNodes() ([]NodeInfo, error) { if strings.HasPrefix(n.Vnfs, "docker://") { //TODO: This is a kludge and shouldn't be done here. We need to go back // and do this from a "vnfs" interface or package - n.Vnfs = LocalStateDir + "/oci/vnfs/name/" + path.Base(n.Vnfs) + n.VnfsDir = LocalStateDir + "/oci/vnfs/name/" + path.Base(n.Vnfs) + } else { + n.VnfsDir = n.Vnfs } ret = append(ret, n) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 887d46b3..1717cdd9 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -9,19 +9,22 @@ import ( // THIS IS NOT BEING USED YET AND IS THUS A WORK IN PROGRESS -const ConfigFile = "/etc/warewulf/warewulf.conf" +const SysConfDir = "/etc/warewulf/" +const LocalStateDir = "/var/warewulf" type Config struct { - Port int `yaml:"warewulfd port", envconfig:"WAREWULFD_PORT"` - Ipaddr string `yaml:"warewulfd ipaddr", envconfig:"WAREWULFD_IPADDR"` - InsecureRuntime bool `yaml:"insecure runtime"` - Debug bool `yaml:"debug"` + Port int `yaml:"warewulfd port", envconfig:"WAREWULFD_PORT"` + Ipaddr string `yaml:"warewulfd ipaddr", envconfig:"WAREWULFD_IPADDR"` + InsecureRuntime bool `yaml:"insecure runtime"` + Debug bool `yaml:"debug"` + SysConfDir string `yaml:"system config dir"` + LocalStateDir string `yaml:"local state dir"` } func New() (Config, error) { var c Config - fd, err := ioutil.ReadFile(ConfigFile) + fd, err := ioutil.ReadFile(SysConfDir + "warewulf.conf") if err != nil { return c, err } @@ -37,7 +40,14 @@ func New() (Config, error) { } if c.Ipaddr == "" { - fmt.Printf("ERROR: 'warewulf ipaddr' has not been set in %s\n", ConfigFile) + fmt.Printf("ERROR: 'warewulf ipaddr' has not been set in %s\n", SysConfDir+"warewulf.conf") + } + + if c.SysConfDir == "" { + c.SysConfDir = SysConfDir + } + if c.LocalStateDir == "" { + c.LocalStateDir = LocalStateDir } return c, nil diff --git a/internal/pkg/oci/puller.go b/internal/pkg/oci/puller.go index e7e7c4ee..f04704a9 100644 --- a/internal/pkg/oci/puller.go +++ b/internal/pkg/oci/puller.go @@ -126,8 +126,8 @@ func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { // copy to cache location _, err = copy.Image(ctx, policyCtx, cacheRef, srcRef, ©.Options{ - ReportWriter: os.Stdout, - SourceCtx: p.sysCtx, + //ReportWriter: os.Stdout, + SourceCtx: p.sysCtx, }) if err != nil { return err diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go new file mode 100644 index 00000000..5d7c01ea --- /dev/null +++ b/internal/pkg/vnfs/vnfs.go @@ -0,0 +1,51 @@ +package vnfs + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "path" + "strings" +) + +type VnfsObject struct { + Source string + RootPath string + ImagePath string +} + +func New(s string) VnfsObject { + var ret VnfsObject + + ret.Source = s + + return ret +} + +func (self *VnfsObject) Name() string { + if self.Source == "" { + return "" + } + + if strings.HasPrefix(self.Source, "/") { + return path.Base(self.Source) + } + + uri := strings.Split(self.Source, "://") + + return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") +} + +func (self *VnfsObject) Image() string { + if self.Source == "" { + return "" + } + + return config.LocalStateDir + "/provision/vnfs/" + self.Name() + ".img.gz" +} + +func (self *VnfsObject) Root() string { + if self.Source == "" { + return "" + } + + return config.LocalStateDir + "/chroots/" + self.Name() +} diff --git a/overlays/runtime/default/etc/group.in b/overlays/runtime/default/etc/group.in index a16792ab..e1ba0627 100644 --- a/overlays/runtime/default/etc/group.in +++ b/overlays/runtime/default/etc/group.in @@ -1,2 +1,2 @@ -#WWINCLUDE @VNFS@/etc/group -#WWINCLUDE /etc/group \ No newline at end of file +#WWINCLUDE @VNFSDIR@/etc/group +#WWINCLUDE /etc/group diff --git a/overlays/runtime/default/etc/passwd.in b/overlays/runtime/default/etc/passwd.in index 3e298b08..22100da3 100644 --- a/overlays/runtime/default/etc/passwd.in +++ b/overlays/runtime/default/etc/passwd.in @@ -1,3 +1,3 @@ root::0:0:root:/root:/bin/bash -#WWINCLUDE @VNFS@/etc/passwd -#WWINCLUDE /etc/passwd \ No newline at end of file +#WWINCLUDE @VNFSDIR@/etc/passwd +#WWINCLUDE /etc/passwd From 0c369270ef8820fad440ebdc6c5996a09e1290f3 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Mon, 9 Nov 2020 20:42:12 -0800 Subject: [PATCH 02/15] Beginning to break out wwbuild into wwctl with proper Go structure and Cobra --- cmd/wwctl/main.go | 13 +++ go.mod | 1 + go.sum | 182 +++++++++++++++++++++++++++++++ internal/app/wwctl/build/root.go | 35 ++++++ internal/app/wwctl/root.go | 43 ++++++++ internal/pkg/wwlog/wwlog.go | 59 ++++++++++ 6 files changed, 333 insertions(+) create mode 100644 cmd/wwctl/main.go create mode 100644 internal/app/wwctl/build/root.go create mode 100644 internal/app/wwctl/root.go create mode 100644 internal/pkg/wwlog/wwlog.go diff --git a/cmd/wwctl/main.go b/cmd/wwctl/main.go new file mode 100644 index 00000000..8f72ea22 --- /dev/null +++ b/cmd/wwctl/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "github.com/hpcng/warewulf/internal/app/wwctl" +) + + +func main() { + + root := wwctl.GetRootCommand() + + root.Execute() +} \ No newline at end of file diff --git a/go.mod b/go.mod index 36d5b302..f2c34b44 100644 --- a/go.mod +++ b/go.mod @@ -7,5 +7,6 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/opencontainers/umoci v0.4.6 + github.com/spf13/cobra v1.1.1 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index fb36feea..f5191aba 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,25 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -17,13 +31,19 @@ github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE github.com/apex/logs v0.0.7/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -44,8 +64,12 @@ github.com/containers/ocicrypt v1.0.3 h1:vYgl+RZ9Q3DPMuTfxmN+qp0X2Bj52uuY2vnt6Gz github.com/containers/ocicrypt v1.0.3/go.mod h1:CUBa+8MRNL/VkpxYIpaMtgn1WgXGyvPQj8jcy0EVG6g= github.com/containers/storage v1.23.6 h1:3rcZ1KTNv8q7SkZ75gcrFGYqTeiuI04Zg7m9X1sCg/s= github.com/containers/storage v1.23.6/go.mod h1:haFs0HRowKwyzvWEx9EgI3WsL8XCSnBDb5f8P5CAxJY= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -55,6 +79,8 @@ github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f h1:Sm8iD2lifO31DwXfkGzq8VgA7rwxPjRsYmeo0K/dF9Y= @@ -76,6 +102,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -86,7 +113,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -99,6 +129,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -106,19 +138,53 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= @@ -142,10 +208,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -157,7 +226,16 @@ github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lL github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/moby/sys/mountinfo v0.3.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -173,6 +251,7 @@ github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgi github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -196,15 +275,19 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi github.com/opencontainers/umoci v0.4.6 h1:nUULYM+jSLLJCVN2gd4wldm8/yuVMahC36UXna3jEqI= github.com/opencontainers/umoci v0.4.6/go.mod h1:ZyTwgJPvYl9xv1Cf3ykU41BxMSCxW3AtoueA5Bmxs1E= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -212,19 +295,27 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4= github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -237,9 +328,22 @@ github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -247,6 +351,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= @@ -256,6 +361,7 @@ github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pv github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -277,31 +383,60 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -309,6 +444,8 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -317,17 +454,24 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -345,8 +489,11 @@ golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -354,18 +501,44 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -385,12 +558,17 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -400,4 +578,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclp gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/app/wwctl/build/root.go b/internal/app/wwctl/build/root.go new file mode 100644 index 00000000..4eee8699 --- /dev/null +++ b/internal/app/wwctl/build/root.go @@ -0,0 +1,35 @@ +package build + +import ( + "fmt" + "github.com/spf13/cobra" +) + +var ( + buildCmd = &cobra.Command{ + Use: "build", + Short: "Warewulf build subcommand", + Long: "Warewulf build is used to build VNFS, kernel, and overlay objects for provisioning.", + RunE: buildRunE, + } + option string +) + +func init() { + //buildCmd.PersistentFlags().StringVar(&flagHost, FlagHost, "localhost:7331", "Address of the Fuzzball Host.") + + buildCmd.PersistentFlags().StringVarP(&option, "testopt", "t", "default", "This is a test option") + +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetCommand() *cobra.Command { + return buildCmd +} + +func buildRunE(cmd *cobra.Command, args []string) error { + fmt.Printf("Hello World. Got Option: %s, %s\n", option, args) + + return nil +} + diff --git a/internal/app/wwctl/root.go b/internal/app/wwctl/root.go new file mode 100644 index 00000000..f0b736fb --- /dev/null +++ b/internal/app/wwctl/root.go @@ -0,0 +1,43 @@ +package wwctl + +import ( + "github.com/hpcng/warewulf/internal/app/wwctl/build" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "wwctl", + Short: "Warewulf CTL", + Long: "Fuzzball CLI is an application for interacting with a Fuzzball Service.", + PersistentPreRunE: rootPersistentPreRunE, + } + verboseArg bool + debugArg bool +) + + + +func init() { + rootCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.") + rootCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.") + + rootCmd.AddCommand(build.GetCommand()) +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetRootCommand() *cobra.Command { + return rootCmd +} + +func rootPersistentPreRunE(cmd *cobra.Command, args []string) error { + if verboseArg == true { + wwlog.SetLevel(wwlog.VERBOSE) + } else if debugArg == true { + wwlog.SetLevel(wwlog.DEBUG) + } else { + wwlog.SetLevel(wwlog.INFO) + } + return nil +} \ No newline at end of file diff --git a/internal/pkg/wwlog/wwlog.go b/internal/pkg/wwlog/wwlog.go new file mode 100644 index 00000000..44feb1a8 --- /dev/null +++ b/internal/pkg/wwlog/wwlog.go @@ -0,0 +1,59 @@ +package wwlog + +import ( + "log" +) + +const ( + CRITICAL = 0 + ERROR = 1 + WARN = 2 + INFO = 3 + VERBOSE = 4 + DEBUG = 5 +) + +var ( + logLevel uint +) + +func SetLevel(level uint) { + logLevel = level + + if logLevel == DEBUG { + log.SetFlags(log.Lmicroseconds | log.Llongfile) + } else { + log.SetFlags(0) + } + Printf(VERBOSE, "Set log level to: %d\n", logLevel) +} + +func prefixLevel(level uint) { + if level == DEBUG { + log.SetPrefix("[DEBUG] ") + } else if level == VERBOSE { + log.SetPrefix("[VERBOSE] ") + } else if level == INFO { + log.SetPrefix("[INFO] ") + } else if level == WARN { + log.SetPrefix("[WARN] ") + } else if level == ERROR { + log.SetPrefix("[ERROR] ") + } else if level == CRITICAL { + log.SetPrefix("[CRITICAL] ") + } +} + +func Println(level uint, message string) { + if level <= logLevel { + prefixLevel(level) + log.Println(message) + } +} + +func Printf(level uint, message string, a...interface{}) { + if level <= logLevel { + prefixLevel(level) + log.Printf(message, a...) + } +} From 189618a111d641483cc353fac136ebaa6e90c8d4 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Tue, 10 Nov 2020 21:28:39 -0800 Subject: [PATCH 03/15] Major refactor for Cobra and new pkg/app structure for wwctl --- Makefile | 9 +- cmd/warewulfd/runtime.go | 2 +- cmd/warewulfd/system.go | 2 +- cmd/warewulfd/warewulfd.go | 4 +- cmd/wwbuild/overlay-runtime.go | 6 +- cmd/wwbuild/overlay-system.go | 2 +- cmd/wwbuild/vnfs-local.go | 44 ------ cmd/wwbuild/wwbuild.go | 9 +- cmd/wwclient/wwclient.go | 6 +- cmd/wwctl/main.go | 1 - internal/app/wwctl/build/build.go | 79 +++++++++++ internal/app/wwctl/build/kernel/kernel.go | 74 ++++++++++ internal/app/wwctl/build/root.go | 30 ++-- .../build/runtime-overlay/runtime-overlay.go | 133 ++++++++++++++++++ .../build/system-overlay/system-overlay.go | 133 ++++++++++++++++++ internal/app/wwctl/build/vnfs/containerdir.go | 50 +++++++ .../app/wwctl/build/vnfs/docker.go | 32 ++--- .../app/wwctl/build/vnfs/util.go | 7 +- internal/app/wwctl/build/vnfs/vnfs.go | 51 +++++++ internal/app/wwctl/root.go | 6 +- internal/pkg/assets/assets.go | 8 +- internal/pkg/util/util.go | 18 +++ internal/pkg/vnfs/vnfs.go | 52 +++++-- internal/pkg/wwlog/wwlog.go | 30 ++-- 24 files changed, 658 insertions(+), 130 deletions(-) delete mode 100644 cmd/wwbuild/vnfs-local.go create mode 100644 internal/app/wwctl/build/build.go create mode 100644 internal/app/wwctl/build/kernel/kernel.go create mode 100644 internal/app/wwctl/build/runtime-overlay/runtime-overlay.go create mode 100644 internal/app/wwctl/build/system-overlay/system-overlay.go create mode 100644 internal/app/wwctl/build/vnfs/containerdir.go rename cmd/wwbuild/vnfs-oci.go => internal/app/wwctl/build/vnfs/docker.go (66%) rename cmd/wwbuild/vnfs.go => internal/app/wwctl/build/vnfs/util.go (86%) create mode 100644 internal/app/wwctl/build/vnfs/vnfs.go diff --git a/Makefile b/Makefile index 9247e807..f792b785 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ lint: @echo Running golangci-lint... @$(GOLANGCI_LINT) run --build-tags "$(WW_BUILD_GO_BUILD_TAGS)" ./... -all: warewulfd wwbuild wwclient +all: warewulfd wwctl wwclient files: all install -d -m 0755 /var/warewulf/ @@ -67,8 +67,8 @@ services: files warewulfd: cd cmd/warewulfd; go build -mod vendor -o ../../warewulfd -wwbuild: - cd cmd/wwbuild; go build -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../wwbuild +wwctl: + cd cmd/wwctl; go build -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../wwctl wwclient: cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags '-extldflags -static' -o ../../wwclient @@ -76,7 +76,6 @@ wwclient: clean: rm -f warewulfd rm -f wwbuild - rm -f wwclient + rm -f wwctl install: files services - diff --git a/cmd/warewulfd/runtime.go b/cmd/warewulfd/runtime.go index 7ee88283..4378e3d7 100644 --- a/cmd/warewulfd/runtime.go +++ b/cmd/warewulfd/runtime.go @@ -59,7 +59,7 @@ func runtimeOverlaySend(w http.ResponseWriter, req *http.Request) { } } else { w.WriteHeader(503) - log.Printf("ERROR: No 'runtime overlay' set for node %s\n", node.Fqdn) + log.Printf("ERROR: No 'runtime system-overlay' set for node %s\n", node.Fqdn) } return diff --git a/cmd/warewulfd/system.go b/cmd/warewulfd/system.go index 7efe428b..3007fdf8 100644 --- a/cmd/warewulfd/system.go +++ b/cmd/warewulfd/system.go @@ -27,7 +27,7 @@ func systemOverlaySend(w http.ResponseWriter, req *http.Request) { } } else { w.WriteHeader(503) - log.Printf("ERROR: No 'system overlay' set for node %s\n", node.Fqdn) + log.Printf("ERROR: No 'system system-overlay' set for node %s\n", node.Fqdn) } return diff --git a/cmd/warewulfd/warewulfd.go b/cmd/warewulfd/warewulfd.go index 1cf34ed0..b9b84552 100644 --- a/cmd/warewulfd/warewulfd.go +++ b/cmd/warewulfd/warewulfd.go @@ -66,8 +66,8 @@ func main() { http.HandleFunc("/kernel/", kernelSend) http.HandleFunc("/kmods/", kmodsSend) http.HandleFunc("/vnfs/", vnfsSend) - http.HandleFunc("/overlay-system/", systemOverlaySend) - http.HandleFunc("/overlay-runtime", runtimeOverlaySend) + http.HandleFunc("/system-overlay-system/", systemOverlaySend) + http.HandleFunc("/system-overlay-runtime", runtimeOverlaySend) http.ListenAndServe(":9873", nil) } diff --git a/cmd/wwbuild/overlay-runtime.go b/cmd/wwbuild/overlay-runtime.go index 6e077c87..1b738b01 100644 --- a/cmd/wwbuild/overlay-runtime.go +++ b/cmd/wwbuild/overlay-runtime.go @@ -19,7 +19,9 @@ func overlayRuntime(node assets.NodeInfo, replace map[string]string, wg *sync.Wa /* destModTime := time.Time{} destMod, err := os.Stat(OverlayFile) - if err == nil { + if err == n + + il { destModTime = destMod.ModTime() } configMod, err := os.Stat("/etc/warewulf/nodes.conf") @@ -43,7 +45,7 @@ func overlayRuntime(node assets.NodeInfo, replace map[string]string, wg *sync.Wa // if sourceModTime.After(destModTime) || configModTime.After(destModTime) { fmt.Printf("RUNTIME: %s\n", node.Fqdn) - overlayDest := "/tmp/.overlay-" + util.RandomString(16) + overlayDest := "/tmp/.system-overlay-" + util.RandomString(16) BuildOverlayDir(OverlayDir, overlayDest, replace) cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) diff --git a/cmd/wwbuild/overlay-system.go b/cmd/wwbuild/overlay-system.go index 0cfaca23..1fbb0f23 100644 --- a/cmd/wwbuild/overlay-system.go +++ b/cmd/wwbuild/overlay-system.go @@ -45,7 +45,7 @@ func overlaySystem(node assets.NodeInfo, replace map[string]string, wg *sync.Wai if sourceModTime.After(destModTime) || configModTime.After(destModTime) { fmt.Printf("SYSTEM: %s\n", node.Fqdn) - overlayDest := "/tmp/.overlay-" + util.RandomString(16) + overlayDest := "/tmp/.system-overlay-" + util.RandomString(16) BuildOverlayDir(OverlayDir, overlayDest, replace) cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) diff --git a/cmd/wwbuild/vnfs-local.go b/cmd/wwbuild/vnfs-local.go deleted file mode 100644 index 91e5babb..00000000 --- a/cmd/wwbuild/vnfs-local.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "os" - "path" - "sync" -) - -func vnfsLocalBuild(vnfsPath string, wg *sync.WaitGroup) { - defer wg.Done() - v := vnfs.New(vnfsPath) - - if _, err := os.Stat(vnfsPath); err == nil { - // TODO: Build VNFS to temporary file and move to real location when complete atomically - // TODO: Check time stamps of sourcedir and build file to see if we need to rebuild or skip - - //vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(vnfsPath)) - - fmt.Printf("Building local Container: %s\n", vnfsPath) - - err := os.MkdirAll(path.Dir(v.Image()), 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - - err = buildVnfs(vnfsPath, v.Image()) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - os.Exit(1) - } - - err = buildLinks(v, vnfsPath) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - os.Exit(1) - } - - } else { - fmt.Printf("SKIPPING VNFS: (bad path) %s\n", vnfsPath) - } -} diff --git a/cmd/wwbuild/wwbuild.go b/cmd/wwbuild/wwbuild.go index 8daa7e4f..bc4bcced 100644 --- a/cmd/wwbuild/wwbuild.go +++ b/cmd/wwbuild/wwbuild.go @@ -1,5 +1,10 @@ package main +import "os" + +import "os" + + import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/assets" @@ -16,7 +21,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Printf("USAGE: %s [vnfs/kernel/overlay] (node regex)\n", os.Args[0]) + fmt.Printf("USAGE: %s [vnfs/kernel/system-overlay] (node regex)\n", os.Args[0]) return } @@ -107,7 +112,7 @@ func main() { } } } - } else if os.Args[1] == "overlay" { + } else if os.Args[1] == "system-overlay" { var nodeList []assets.NodeInfo var wg sync.WaitGroup diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go index fab560f3..0f71894b 100644 --- a/cmd/wwclient/wwclient.go +++ b/cmd/wwclient/wwclient.go @@ -19,7 +19,7 @@ func main() { time.Sleep(5000 * time.Millisecond) } else { fmt.Printf("Called via: %s\n", os.Args[0]) - fmt.Printf("Runtime overlay is being put in '/warewulf/wwclient-test' rather than '/'\n") + fmt.Printf("Runtime system-overlay is being put in '/warewulf/wwclient-test' rather than '/'\n") os.MkdirAll("/warewulf/wwclient-test", 0755) os.Chdir("/warewulf/wwclient-test") } @@ -61,7 +61,7 @@ func main() { for true { var err error - getString := fmt.Sprintf("http://%s:%d/overlay-runtime", config.Ipaddr, config.Port) + getString := fmt.Sprintf("http://%s:%d/system-overlay-runtime", config.Ipaddr, config.Port) resp, err = webclient.Get(getString) if err == nil { break @@ -84,7 +84,7 @@ func main() { if resp.StatusCode != 200 { - log.Printf("Not updating runtime overlay, got status code: %d\n", resp.StatusCode) + log.Printf("Not updating runtime system-overlay, got status code: %d\n", resp.StatusCode) time.Sleep(60000 * time.Millisecond) continue } diff --git a/cmd/wwctl/main.go b/cmd/wwctl/main.go index 8f72ea22..a8718e31 100644 --- a/cmd/wwctl/main.go +++ b/cmd/wwctl/main.go @@ -6,7 +6,6 @@ import ( func main() { - root := wwctl.GetRootCommand() root.Execute() diff --git a/internal/app/wwctl/build/build.go b/internal/app/wwctl/build/build.go new file mode 100644 index 00000000..323c6095 --- /dev/null +++ b/internal/app/wwctl/build/build.go @@ -0,0 +1,79 @@ +package build + +import ( + "github.com/hpcng/warewulf/internal/app/wwctl/build/kernel" + runtime_overlay "github.com/hpcng/warewulf/internal/app/wwctl/build/runtime-overlay" + system_overlay "github.com/hpcng/warewulf/internal/app/wwctl/build/system-overlay" + "github.com/hpcng/warewulf/internal/app/wwctl/build/vnfs" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" + "os" +) + + +func CobraRunE(cmd *cobra.Command, args []string) error { + var nodeList []assets.NodeInfo + + if buildAll == true { + wwlog.Printf(wwlog.VERBOSE, "Building all components\n") + buildVnfs = true + buildKernel = true + buildSystemOverlay = true; + buildRuntimeOverlay = true; + } + + if len(args) >= 1 { + nodeList, _ = assets.SearchByName(args[0]) + } else { + nodeList, _ = assets.FindAllNodes() + } + + if len(nodeList) == 0 { + wwlog.Printf(wwlog.ERROR, "No nodes found matching: '%s'\n", args[0]) + os.Exit(255) + } else { + wwlog.Printf(wwlog.VERBOSE, "Found matching nodes for build: %d\n", len(nodeList)) + } + + if buildVnfs == true { +// wwlog.Printf(wwlog.INFO, "===============================================================================\n") + + err := vnfs.Build(nodeList, buildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(255) + } + } + + if buildKernel == true { +// wwlog.Printf(wwlog.INFO, "===============================================================================\n") + + err := kernel.Build(nodeList, buildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(255) + } + } + + if buildSystemOverlay == true { +// wwlog.Printf(wwlog.INFO, "===============================================================================\n") + err := system_overlay.Build(nodeList, buildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(255) + } + } + + if buildRuntimeOverlay == true { +// wwlog.Printf(wwlog.INFO, "===============================================================================\n") + err := runtime_overlay.Build(nodeList, buildForce) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(255) + } + } + + + return nil +} \ No newline at end of file diff --git a/internal/app/wwctl/build/kernel/kernel.go b/internal/app/wwctl/build/kernel/kernel.go new file mode 100644 index 00000000..3023264f --- /dev/null +++ b/internal/app/wwctl/build/kernel/kernel.go @@ -0,0 +1,74 @@ +package kernel + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "os" + "os/exec" + "path" +) + +const kernelProvisionPath = "/provision/kernel/" + +func Build(nodeList []assets.NodeInfo, force bool) error { + set := make(map[string]int) + + wwlog.Printf(wwlog.INFO, "Importing Kernels:\n") + wwlog.SetIndent(4) + + for _, node := range nodeList { + if node.KernelVersion != "" { + set[node.KernelVersion] ++ + wwlog.Printf(wwlog.DEBUG, "Node '%s' has KernelVersion '%s'\n", node.Fqdn, node.KernelVersion) + } + } + + for kernelVersion := range set { + kernelImage := "/boot/vmlinuz-"+kernelVersion + kernelDrivers := "/lib/modules/"+kernelVersion + kernelDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "vmlinuz-"+kernelVersion) + driversDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "kmods-"+kernelVersion+".img") + + + // Create the kernel destination path just in case it doesn't exist + os.MkdirAll(path.Join(config.LocalStateDir, kernelProvisionPath), 0755) + + if _, err := os.Stat(kernelImage); err == nil { + if util.PathIsNewer(kernelImage, kernelDestination) && force == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, kernel is current\n", "vmlinuz-"+kernelVersion) + + } else { + err := util.CopyFile(kernelImage, kernelDestination) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Failed copying kernel image: %s\n", err) + continue + } + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", "vmlinuz-"+kernelVersion) + + } + } + + if _, err := os.Stat(kernelDrivers); err == nil { + if util.PathIsNewer(kernelDrivers, driversDestination) && force == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, kernel Drivers are current\n", "kmods-"+kernelVersion+".img") + + } else { + cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelDrivers, driversDestination) + err := exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not generate kernel driver overlay: %s\n", err) + continue + } + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", "kmods-"+kernelVersion+".img") + } + } + + } + + wwlog.SetIndent(0) + + return nil +} \ No newline at end of file diff --git a/internal/app/wwctl/build/root.go b/internal/app/wwctl/build/root.go index 4eee8699..10244ce9 100644 --- a/internal/app/wwctl/build/root.go +++ b/internal/app/wwctl/build/root.go @@ -1,7 +1,6 @@ package build import ( - "fmt" "github.com/spf13/cobra" ) @@ -9,16 +8,26 @@ var ( buildCmd = &cobra.Command{ Use: "build", Short: "Warewulf build subcommand", - Long: "Warewulf build is used to build VNFS, kernel, and overlay objects for provisioning.", - RunE: buildRunE, + Long: "Warewulf build is used to build VNFS, kernel, and system-overlay objects for\n" + + "provisioning. The default usage will be to build and/or update anything\n" + + "that seems to be needed.", + RunE: CobraRunE, } - option string + buildVnfs bool + buildKernel bool + buildRuntimeOverlay bool + buildSystemOverlay bool + buildAll bool + buildForce bool ) func init() { - //buildCmd.PersistentFlags().StringVar(&flagHost, FlagHost, "localhost:7331", "Address of the Fuzzball Host.") - - buildCmd.PersistentFlags().StringVarP(&option, "testopt", "t", "default", "This is a test option") + buildCmd.PersistentFlags().BoolVarP(&buildVnfs, "vnfs", "V", false, "Build and/or update VNFS images.") + buildCmd.PersistentFlags().BoolVarP(&buildKernel, "kernel", "K", false, "Build and/or update Kernel images.") + buildCmd.PersistentFlags().BoolVarP(&buildRuntimeOverlay, "runtime", "R", false, "Build and/or update runtime overlays") + buildCmd.PersistentFlags().BoolVarP(&buildSystemOverlay, "system", "S", false, "Build and/or update system overlays") + buildCmd.PersistentFlags().BoolVarP(&buildAll, "all", "A", false, "Build and/or update all components") + buildCmd.PersistentFlags().BoolVarP(&buildForce, "force", "f", false, "Force build even if nothing has been updated.") } @@ -26,10 +35,3 @@ func init() { func GetCommand() *cobra.Command { return buildCmd } - -func buildRunE(cmd *cobra.Command, args []string) error { - fmt.Printf("Hello World. Got Option: %s, %s\n", option, args) - - return nil -} - diff --git a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go new file mode 100644 index 00000000..6e553eee --- /dev/null +++ b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go @@ -0,0 +1,133 @@ +package runtime_overlay + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "text/template" +) + +func Build(nodeList []assets.NodeInfo, force bool) error { + wwlog.Printf(wwlog.INFO, "Building Runtime Overlays:\n") + wwlog.SetIndent(4) + + for _, node := range nodeList { + if node.RuntimeOverlay != "" { + OverlayDir := path.Join(config.LocalStateDir, "/overlays/runtime/", node.RuntimeOverlay) + OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/runtime/", node.Fqdn+".img") + + wwlog.Printf(wwlog.VERBOSE, "Building Runtime Overlay for: %s\n", node.Fqdn) + + tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-runtime-overlay-") + if err != nil { + return err + } + + if _, err := os.Stat(OverlayDir); err != nil { + return err + } + + err = os.MkdirAll(path.Dir(OverlayFile), 0755) + if err != nil { + return err + } + + if force == false { + wwlog.Printf(wwlog.DEBUG, "Checking if overlay is required\n") + } + if util.PathIsNewer(OverlayDir, OverlayFile) { + if force == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, overlay is current\n", node.Fqdn) + continue + } + } + + wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlayDir) + err = os.Chdir(OverlayDir) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not chdir() to OverlayDir: %s\n", OverlayDir) + continue + } + + wwlog.Printf(wwlog.DEBUG, "Walking the file system: %s\n", OverlayDir) + err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) + + err := os.MkdirAll(path.Join(tmpDir, location), info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if filepath.Ext(location) == ".ww" { + wwlog.Printf(wwlog.DEBUG, "Found template file: %s\n", location) + + destFile := strings.TrimSuffix(location, ".ww") + + tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*") + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + w, err := os.OpenFile(path.Join(tmpDir, destFile), os.O_RDWR|os.O_CREATE, info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + defer w.Close() + + err = tmpl.Execute(w, node) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b == true { + wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location) + } else { + wwlog.Printf(wwlog.DEBUG, "Found file: %s\n", location) + + err := util.CopyFile(path.Join(OverlayDir, location), path.Join(tmpDir, location)) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } + + return nil + }) + wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", node.Fqdn) + + cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, OverlayFile) + wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not generate runtime image overlay: %s\n", err) + continue + } + wwlog.Printf(wwlog.INFO, "%-3s: Done\n", node.Fqdn) + + wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir) + os.RemoveAll(tmpDir) + } + } + + wwlog.SetIndent(0) + return nil +} diff --git a/internal/app/wwctl/build/system-overlay/system-overlay.go b/internal/app/wwctl/build/system-overlay/system-overlay.go new file mode 100644 index 00000000..e20d1e1c --- /dev/null +++ b/internal/app/wwctl/build/system-overlay/system-overlay.go @@ -0,0 +1,133 @@ +package system_overlay + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "text/template" +) + +func Build(nodeList []assets.NodeInfo, force bool) error { + wwlog.Printf(wwlog.INFO, "Building System Overlays:\n") + wwlog.SetIndent(4) + + for _, node := range nodeList { + if node.SystemOverlay != "" { + OverlayDir := path.Join(config.LocalStateDir, "/overlays/system/", node.SystemOverlay) + OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/system/", node.Fqdn+".img") + + wwlog.Printf(wwlog.VERBOSE, "Building System Overlay for: %s\n", node.Fqdn) + + tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-system-overlay-") + if err != nil { + return err + } + + if _, err := os.Stat(OverlayDir); err != nil { + return err + } + + err = os.MkdirAll(path.Dir(OverlayFile), 0755) + if err != nil { + return err + } + + if force == false { + wwlog.Printf(wwlog.DEBUG, "Checking if overlay is required\n") + } + if util.PathIsNewer(OverlayDir, OverlayFile) { + if force == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, overlay is current\n", node.Fqdn) + continue + } + } + + wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlayDir) + err = os.Chdir(OverlayDir) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not chdir() to OverlayDir: %s\n", OverlayDir) + continue + } + + wwlog.Printf(wwlog.DEBUG, "Walking the file system: %s\n", OverlayDir) + err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) + + err := os.MkdirAll(path.Join(tmpDir, location), info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if filepath.Ext(location) == ".ww" { + wwlog.Printf(wwlog.DEBUG, "Found template file: %s\n", location) + + destFile := strings.TrimSuffix(location, ".ww") + + tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*") + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + w, err := os.OpenFile(path.Join(tmpDir, destFile), os.O_RDWR|os.O_CREATE, info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + defer w.Close() + + err = tmpl.Execute(w, node) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b == true { + wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location) + } else { + wwlog.Printf(wwlog.DEBUG, "Found file: %s\n", location) + + err := util.CopyFile(path.Join(OverlayDir, location), path.Join(tmpDir, location)) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } + + return nil + }) + wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", node.Fqdn) + + cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, OverlayFile) + wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not generate system image overlay: %s\n", err) + continue + } + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", node.Fqdn) + + wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir) + os.RemoveAll(tmpDir) + } + } + + wwlog.SetIndent(0) + return nil +} diff --git a/internal/app/wwctl/build/vnfs/containerdir.go b/internal/app/wwctl/build/vnfs/containerdir.go new file mode 100644 index 00000000..b3cb6be4 --- /dev/null +++ b/internal/app/wwctl/build/vnfs/containerdir.go @@ -0,0 +1,50 @@ +package vnfs + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "os" + "path" +) + +func BuildContainerdir(v vnfs.VnfsObject) { + + if _, err := os.Stat(v.Source()); err != nil { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping (bad path)\n", v.Name()) + return + } + + wwlog.Printf(wwlog.DEBUG, "Checking if there have been any updates to the VNFS directory\n") + if util.PathIsNewer(v.Source(), v.Image()) { + if buildForce == false { + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name()) + return + } + } + + wwlog.Printf(wwlog.DEBUG, "Making the directory: %s\n", path.Dir(v.Image())) + err := os.MkdirAll(path.Dir(v.Image()), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + + wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", v.Source(), v.Image()) + err = buildVnfs(v.Source(), v.Image()) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + os.Exit(1) + } + + wwlog.Printf(wwlog.DEBUG, "Building links for Warewulf access to chroot\n") + err = buildLinks(v, v.Source()) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + os.Exit(1) + } + + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", v.Name()) + +} diff --git a/cmd/wwbuild/vnfs-oci.go b/internal/app/wwctl/build/vnfs/docker.go similarity index 66% rename from cmd/wwbuild/vnfs-oci.go rename to internal/app/wwctl/build/vnfs/docker.go index 399fbc0c..79ea8825 100644 --- a/cmd/wwbuild/vnfs-oci.go +++ b/internal/app/wwctl/build/vnfs/docker.go @@ -1,4 +1,4 @@ -package main +package vnfs import ( "context" @@ -6,10 +6,9 @@ import ( "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/oci" "github.com/hpcng/warewulf/internal/pkg/vnfs" - "log" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "os" "path" - "sync" ) const ( @@ -17,21 +16,17 @@ const ( VnfsHashDir = config.LocalStateDir + "/oci/vnfs/hash/" ) -func vnfsOciBuild(OciPath string, wg *sync.WaitGroup) { - v := vnfs.New(OciPath) - - // vnfsDestination := fmt.Sprintf("%s/provision/vnfs/%s.img.gz", LocalStateDir, path.Base(OciPath)) - defer wg.Done() - - log.Printf("Building OCI Container: %s\n", OciPath) +func BuildDocker(v vnfs.VnfsObject) { + wwlog.Printf(wwlog.VERBOSE, "Building OCI Container: %s\n", v.Source()) c, err := oci.NewCache(oci.OptSetCachePath(OciCacheDir)) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } - log.Printf("Downloading OCI container layers\n") - sourcePath, err := c.Pull(context.Background(), OciPath, nil) + + wwlog.Printf(wwlog.VERBOSE, "Downloading OCI container layers\n") + sourcePath, err := c.Pull(context.Background(), v.Source(), nil) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) @@ -42,7 +37,7 @@ func vnfsOciBuild(OciPath string, wg *sync.WaitGroup) { name, err := os.Readlink(v.Image()) if err == nil { if name == hashDestination { - log.Printf("Container already built, no update available\n") + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name()) return } } @@ -63,7 +58,7 @@ func vnfsOciBuild(OciPath string, wg *sync.WaitGroup) { return } - log.Printf("Building bootable VNFS image\n") + wwlog.Printf(wwlog.VERBOSE, "Building bootable VNFS image\n") err = buildVnfs(sourcePath, hashDestination) if err != nil { @@ -71,23 +66,24 @@ func vnfsOciBuild(OciPath string, wg *sync.WaitGroup) { os.Exit(1) } - log.Printf("Finalizing Build\n") + wwlog.Printf(wwlog.VERBOSE, "Finalizing Build\n") _ = os.Remove(v.Image() + "-link") err = os.Symlink(hashDestination, v.Image()+"-link") if err != nil { - fmt.Printf("ERROR: %s\n", err) + wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } err = os.Rename(v.Image()+"-link", v.Image()) err = buildLinks(v, sourcePath) if err != nil { - fmt.Printf("ERROR: %s\n", err) + wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } - log.Printf("Completed building VNFS: %s\n", path.Base(OciPath)) + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", v.Name()) return } + diff --git a/cmd/wwbuild/vnfs.go b/internal/app/wwctl/build/vnfs/util.go similarity index 86% rename from cmd/wwbuild/vnfs.go rename to internal/app/wwctl/build/vnfs/util.go index f9ab7f67..8bd44e3d 100644 --- a/cmd/wwbuild/vnfs.go +++ b/internal/app/wwctl/build/vnfs/util.go @@ -1,10 +1,12 @@ -package main +package vnfs + import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/vnfs" "os" "os/exec" + "path" ) func buildVnfs(source string, dest string) error { @@ -20,6 +22,9 @@ func buildLinks(v vnfs.VnfsObject, source string) error { // Just incase the temporary link location is present, remove it if we can _ = os.Remove(v.Root() + "-link") + // Just incase the directory doesn't exist, make it + _ = os.MkdirAll(path.Dir(v.Root()), 0755) + // Link to a temporary location so we can atomically move the link into place err := os.Symlink(source, v.Root()+"-link") if err != nil { diff --git a/internal/app/wwctl/build/vnfs/vnfs.go b/internal/app/wwctl/build/vnfs/vnfs.go new file mode 100644 index 00000000..aa044208 --- /dev/null +++ b/internal/app/wwctl/build/vnfs/vnfs.go @@ -0,0 +1,51 @@ +package vnfs + +import ( + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "strings" +) + +var buildForce bool + +func Build(nodeList []assets.NodeInfo, force bool) (error) { + set := make(map[string]int) + + wwlog.Printf(wwlog.INFO, "Importing VNFS Images:\n") + wwlog.SetIndent(4) + + buildForce = force + + for _, node := range nodeList { + if node.Vnfs != "" { + set[node.Vnfs] ++ + wwlog.Printf(wwlog.DEBUG, "Node '%s' has VNFS '%s'\n", node.Fqdn, node.Vnfs) + } + } + + for uri := range set { + v := vnfs.New(uri) + wwlog.Printf(wwlog.VERBOSE, "VNFS found: %s (nodes: %d)\n", uri, set[uri]) + if strings.HasPrefix(uri, "docker://") { + BuildDocker(v) + + } else if strings.HasPrefix(uri, "docker-daemon://") { + //wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service: %s\n", uri) + wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service is not supported yet: %s\n", uri) + + } else if strings.HasPrefix(uri, "/") { + if strings.HasSuffix(uri, "tar.gz") { + //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) + wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", uri) + + } else { + BuildContainerdir(v) + } + } + } + + wwlog.SetIndent(0) + + return nil +} \ No newline at end of file diff --git a/internal/app/wwctl/root.go b/internal/app/wwctl/root.go index f0b736fb..bd214392 100644 --- a/internal/app/wwctl/root.go +++ b/internal/app/wwctl/root.go @@ -9,16 +9,14 @@ import ( var ( rootCmd = &cobra.Command{ Use: "wwctl", - Short: "Warewulf CTL", - Long: "Fuzzball CLI is an application for interacting with a Fuzzball Service.", + Short: "Warewulf Control", + Long: "Control interface to the Cluster Warewulf Provisioning System.", PersistentPreRunE: rootPersistentPreRunE, } verboseArg bool debugArg bool ) - - func init() { rootCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.") rootCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.") diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index eb08c4a3..6c19acdf 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -31,8 +31,8 @@ type nodeGroup struct { Comment string Vnfs string Ipxe string `yaml:"ipxe template"` - SystemOverlay string `yaml:"system overlay""` - RuntimeOverlay string `yaml:"runtime overlay""` + SystemOverlay string `yaml:"system system-overlay""` + RuntimeOverlay string `yaml:"runtime system-overlay""` DomainSuffix string `yaml:"domain suffix"` KernelVersion string `yaml:"kernel version"` Nodes map[string]nodeEntry @@ -42,8 +42,8 @@ type nodeEntry struct { Hostname string Vnfs string Ipxe string `yaml:"ipxe template"` - SystemOverlay string `yaml:"system overlay"` - RuntimeOverlay string `yaml:"runtime overlay"` + SystemOverlay string `yaml:"system system-overlay"` + RuntimeOverlay string `yaml:"runtime system-overlay"` DomainSuffix string `yaml:"domain suffix"` KernelVersion string `yaml:"kernel version"` IpmiIpaddr string `yaml:"ipmi ipaddr"` diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 0fdfca2d..52a3e017 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -1,6 +1,7 @@ package util import ( + "github.com/hpcng/warewulf/internal/pkg/wwlog" "io" "math/rand" "os" @@ -31,6 +32,22 @@ func DirModTime(path string) (time.Time, error) { return lastTime, nil } +func PathIsNewer(source string, compare string) bool { + time1, err := DirModTime(source) + if err != nil { + wwlog.Printf(wwlog.DEBUG, "%s\n", err) + return false + } + + time2, err := DirModTime(compare) + if err != nil { + wwlog.Printf(wwlog.DEBUG, "%s\n", err) + return false + } + + return time1.Before(time2) +} + func RandomString(n int) string { var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -42,6 +59,7 @@ func RandomString(n int) string { } func CopyFile(source string, dest string) error { + wwlog.Printf(wwlog.DEBUG, "Copying '%s' to '%s'\n", source, dest) sourceFD, err := os.Open(source) if err != nil { return err diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 5d7c01ea..2868280e 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -7,45 +7,67 @@ import ( ) type VnfsObject struct { - Source string - RootPath string - ImagePath string + SourcePath string + RootPath string + ImagePath string } func New(s string) VnfsObject { var ret VnfsObject - ret.Source = s + ret.SourcePath = s return ret } + func (self *VnfsObject) Name() string { - if self.Source == "" { + if self.SourcePath == "" { return "" } - if strings.HasPrefix(self.Source, "/") { - return path.Base(self.Source) + if strings.HasPrefix(self.SourcePath, "/") { + return path.Base(self.SourcePath) } - uri := strings.Split(self.Source, "://") - - return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") + return self.SourcePath } -func (self *VnfsObject) Image() string { - if self.Source == "" { +func (self *VnfsObject) NameClean() string { + if self.SourcePath == "" { return "" } - return config.LocalStateDir + "/provision/vnfs/" + self.Name() + ".img.gz" + if strings.HasPrefix(self.SourcePath, "/") { + return path.Base(self.SourcePath) + } + + uri := strings.Split(self.SourcePath, "://") + + return strings.ReplaceAll(uri[1], "/", "_") +} + +func (self *VnfsObject) Source() string { + if self.SourcePath == "" { + return "" + } + + return self.SourcePath +} + + +func (self *VnfsObject) Image() string { + if self.SourcePath == "" { + return "" + } + + return config.LocalStateDir + "/provision/vnfs/" + self.NameClean() + ".img.gz" } func (self *VnfsObject) Root() string { - if self.Source == "" { + if self.SourcePath == "" { return "" } - return config.LocalStateDir + "/chroots/" + self.Name() + return config.LocalStateDir + "/chroots/" + self.NameClean() } diff --git a/internal/pkg/wwlog/wwlog.go b/internal/pkg/wwlog/wwlog.go index 44feb1a8..14f36204 100644 --- a/internal/pkg/wwlog/wwlog.go +++ b/internal/pkg/wwlog/wwlog.go @@ -2,6 +2,7 @@ package wwlog import ( "log" + "strings" ) const ( @@ -15,32 +16,33 @@ const ( var ( logLevel uint + Indent string ) func SetLevel(level uint) { logLevel = level - if logLevel == DEBUG { - log.SetFlags(log.Lmicroseconds | log.Llongfile) - } else { - log.SetFlags(0) - } - Printf(VERBOSE, "Set log level to: %d\n", logLevel) + log.SetFlags(0) + Printf(DEBUG, "Set log level to: %d\n", logLevel) +} + +func SetIndent(i int) { + Indent = strings.Repeat(" ", i) } func prefixLevel(level uint) { if level == DEBUG { - log.SetPrefix("[DEBUG] ") + log.SetPrefix("[DEBUG] "+Indent) } else if level == VERBOSE { - log.SetPrefix("[VERBOSE] ") + log.SetPrefix("[VERBOSE] "+Indent) } else if level == INFO { - log.SetPrefix("[INFO] ") + log.SetPrefix("[INFO] "+Indent) } else if level == WARN { - log.SetPrefix("[WARN] ") + log.SetPrefix("[WARN] "+Indent) } else if level == ERROR { - log.SetPrefix("[ERROR] ") + log.SetPrefix("[ERROR] "+Indent) } else if level == CRITICAL { - log.SetPrefix("[CRITICAL] ") + log.SetPrefix("[CRITICAL] "+Indent) } } @@ -49,6 +51,8 @@ func Println(level uint, message string) { prefixLevel(level) log.Println(message) } + + log.SetPrefix("[LOG] "+Indent) } func Printf(level uint, message string, a...interface{}) { @@ -56,4 +60,6 @@ func Printf(level uint, message string, a...interface{}) { prefixLevel(level) log.Printf(message, a...) } + + log.SetPrefix("[LOG] "+Indent) } From e8c4d41b97afd8ef06b57b1ad2d3ef366bc7dc7d Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 00:09:06 -0800 Subject: [PATCH 04/15] Updates for text/template subsystem integration --- .../build/runtime-overlay/runtime-overlay.go | 11 +++++++++- .../build/system-overlay/system-overlay.go | 11 +++++++++- internal/app/wwctl/build/vnfs/vnfs.go | 2 +- internal/pkg/assets/assets.go | 5 +++-- internal/pkg/assets/template.go | 21 +++++++++++++++++++ internal/pkg/vnfs/vnfs.go | 1 - overlays/runtime/default/etc/group.in | 2 -- overlays/runtime/default/etc/group.ww | 2 ++ overlays/runtime/default/etc/passwd.in | 3 --- overlays/runtime/default/etc/passwd.ww | 3 +++ overlays/runtime/default/etc/ww_hello | 1 - .../default/root/.ssh/authorized_keys.in | 1 - .../default/root/.ssh/authorized_keys.ww | 1 + overlays/system/default/etc/hostname.in | 1 - overlays/system/default/etc/hostname.ww | 1 + .../sysconfig/network-scripts/ifcfg-eth0.in | 12 +++++------ .../system/default/etc/sysconfig/network.in | 2 -- .../system/default/etc/sysconfig/network.ww | 2 ++ .../default/etc/warewulf/warewulf.conf.in | 1 - .../default/etc/warewulf/warewulf.conf.ww | 1 + .../system/default/root/.ssh/authorized_keys | 0 21 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 internal/pkg/assets/template.go delete mode 100644 overlays/runtime/default/etc/group.in create mode 100644 overlays/runtime/default/etc/group.ww delete mode 100644 overlays/runtime/default/etc/passwd.in create mode 100644 overlays/runtime/default/etc/passwd.ww delete mode 100644 overlays/runtime/default/etc/ww_hello delete mode 100644 overlays/runtime/default/root/.ssh/authorized_keys.in create mode 100644 overlays/runtime/default/root/.ssh/authorized_keys.ww delete mode 100644 overlays/system/default/etc/hostname.in create mode 100644 overlays/system/default/etc/hostname.ww delete mode 100644 overlays/system/default/etc/sysconfig/network.in create mode 100644 overlays/system/default/etc/sysconfig/network.ww delete mode 100644 overlays/system/default/etc/warewulf/warewulf.conf.in create mode 100644 overlays/system/default/etc/warewulf/warewulf.conf.ww delete mode 100644 overlays/system/default/root/.ssh/authorized_keys diff --git a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go index 6e553eee..49be8543 100644 --- a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go +++ b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go @@ -16,6 +16,15 @@ import ( "text/template" ) +func fileInclude(path string) string { + wwlog.Printf(wwlog.DEBUG, "Including file into template: %s\n", path) + content, err := ioutil.ReadFile(path) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Template include: %s\n", err) + } + return strings.TrimSuffix(string(content), "\n") +} + func Build(nodeList []assets.NodeInfo, force bool) error { wwlog.Printf(wwlog.INFO, "Building Runtime Overlays:\n") wwlog.SetIndent(4) @@ -78,7 +87,7 @@ func Build(nodeList []assets.NodeInfo, force bool) error { destFile := strings.TrimSuffix(location, ".ww") - tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*") + tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{"Include": fileInclude}).ParseGlob(path.Join(OverlayDir, destFile + ".ww*")) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) return err diff --git a/internal/app/wwctl/build/system-overlay/system-overlay.go b/internal/app/wwctl/build/system-overlay/system-overlay.go index e20d1e1c..f4291747 100644 --- a/internal/app/wwctl/build/system-overlay/system-overlay.go +++ b/internal/app/wwctl/build/system-overlay/system-overlay.go @@ -16,6 +16,15 @@ import ( "text/template" ) +func fileInclude(path string) string { + wwlog.Printf(wwlog.DEBUG, "Including file into template: %s\n", path) + content, err := ioutil.ReadFile(path) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Template include: %s\n", err) + } + return strings.TrimSuffix(string(content), "\n") +} + func Build(nodeList []assets.NodeInfo, force bool) error { wwlog.Printf(wwlog.INFO, "Building System Overlays:\n") wwlog.SetIndent(4) @@ -78,7 +87,7 @@ func Build(nodeList []assets.NodeInfo, force bool) error { destFile := strings.TrimSuffix(location, ".ww") - tmpl, err := template.ParseGlob(path.Join(OverlayDir, destFile) + ".ww*") + tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{"include": fileInclude}).ParseGlob(path.Join(OverlayDir, destFile + ".ww*")) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) return err diff --git a/internal/app/wwctl/build/vnfs/vnfs.go b/internal/app/wwctl/build/vnfs/vnfs.go index aa044208..1fa9add1 100644 --- a/internal/app/wwctl/build/vnfs/vnfs.go +++ b/internal/app/wwctl/build/vnfs/vnfs.go @@ -12,7 +12,7 @@ var buildForce bool func Build(nodeList []assets.NodeInfo, force bool) (error) { set := make(map[string]int) - wwlog.Printf(wwlog.INFO, "Importing VNFS Images:\n") + wwlog.Printf(wwlog.INFO, "Building and Importing VNFS Images:\n") wwlog.SetIndent(4) buildForce = force diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 6c19acdf..8415232d 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -2,9 +2,9 @@ package assets import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "gopkg.in/yaml.v2" "io/ioutil" - "path" "strings" // "os" @@ -140,7 +140,8 @@ func FindAllNodes() ([]NodeInfo, error) { if strings.HasPrefix(n.Vnfs, "docker://") { //TODO: This is a kludge and shouldn't be done here. We need to go back // and do this from a "vnfs" interface or package - n.VnfsDir = LocalStateDir + "/oci/vnfs/name/" + path.Base(n.Vnfs) + v := vnfs.New(n.Vnfs) + n.VnfsDir = v.Root() } else { n.VnfsDir = n.Vnfs } diff --git a/internal/pkg/assets/template.go b/internal/pkg/assets/template.go new file mode 100644 index 00000000..24f43134 --- /dev/null +++ b/internal/pkg/assets/template.go @@ -0,0 +1,21 @@ +package assets + + + +type WWTemplate struct { + GroupName string + HostName string + DomainName string + Fqdn string + Vnfs string + VnfsDir string + Ipxe string + SystemOverlay string + RuntimeOverlay string + KernelVersion string + NetDevs map[string]netDevs +} + +func (t WWTemplate) Import(file string) { + +} \ No newline at end of file diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 2868280e..433afcdf 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -41,7 +41,6 @@ func (self *VnfsObject) NameClean() string { if strings.HasPrefix(self.SourcePath, "/") { return path.Base(self.SourcePath) } - uri := strings.Split(self.SourcePath, "://") return strings.ReplaceAll(uri[1], "/", "_") diff --git a/overlays/runtime/default/etc/group.in b/overlays/runtime/default/etc/group.in deleted file mode 100644 index e1ba0627..00000000 --- a/overlays/runtime/default/etc/group.in +++ /dev/null @@ -1,2 +0,0 @@ -#WWINCLUDE @VNFSDIR@/etc/group -#WWINCLUDE /etc/group diff --git a/overlays/runtime/default/etc/group.ww b/overlays/runtime/default/etc/group.ww new file mode 100644 index 00000000..a0ea3854 --- /dev/null +++ b/overlays/runtime/default/etc/group.ww @@ -0,0 +1,2 @@ +{{Include (printf "%s%s" .VnfsDir "/etc/group")}} +{{Include "/etc/passwd"}} diff --git a/overlays/runtime/default/etc/passwd.in b/overlays/runtime/default/etc/passwd.in deleted file mode 100644 index 22100da3..00000000 --- a/overlays/runtime/default/etc/passwd.in +++ /dev/null @@ -1,3 +0,0 @@ -root::0:0:root:/root:/bin/bash -#WWINCLUDE @VNFSDIR@/etc/passwd -#WWINCLUDE /etc/passwd diff --git a/overlays/runtime/default/etc/passwd.ww b/overlays/runtime/default/etc/passwd.ww new file mode 100644 index 00000000..63234ce6 --- /dev/null +++ b/overlays/runtime/default/etc/passwd.ww @@ -0,0 +1,3 @@ +root::0:0:root:/root:/bin/bash +{{Include (printf "%s%s" .VnfsDir "/etc/passwd")}} +{{Include "/etc/passwd"}} diff --git a/overlays/runtime/default/etc/ww_hello b/overlays/runtime/default/etc/ww_hello deleted file mode 100644 index 6e4a6b48..00000000 --- a/overlays/runtime/default/etc/ww_hello +++ /dev/null @@ -1 +0,0 @@ -Hello from Warewulf diff --git a/overlays/runtime/default/root/.ssh/authorized_keys.in b/overlays/runtime/default/root/.ssh/authorized_keys.in deleted file mode 100644 index f0248c5e..00000000 --- a/overlays/runtime/default/root/.ssh/authorized_keys.in +++ /dev/null @@ -1 +0,0 @@ -#WWINCLUDE /root/.ssh/authorized_keys diff --git a/overlays/runtime/default/root/.ssh/authorized_keys.ww b/overlays/runtime/default/root/.ssh/authorized_keys.ww new file mode 100644 index 00000000..01b8c625 --- /dev/null +++ b/overlays/runtime/default/root/.ssh/authorized_keys.ww @@ -0,0 +1 @@ +{{Include "/root/.ssh/authorized_keys")}} diff --git a/overlays/system/default/etc/hostname.in b/overlays/system/default/etc/hostname.in deleted file mode 100644 index 77515887..00000000 --- a/overlays/system/default/etc/hostname.in +++ /dev/null @@ -1 +0,0 @@ -@HOSTNAME@ diff --git a/overlays/system/default/etc/hostname.ww b/overlays/system/default/etc/hostname.ww new file mode 100644 index 00000000..783b7f08 --- /dev/null +++ b/overlays/system/default/etc/hostname.ww @@ -0,0 +1 @@ +{{.Fqdn}} diff --git a/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in b/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in index f0c09f54..9b27c5af 100644 --- a/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in +++ b/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in @@ -1,8 +1,8 @@ -DEVICE=@ETH0:NAME@ -NAME=@ETH0:NAME@ -IPADDR=@ETH0:IPADDR@ -NETMASK=@ETH0:NETMASK@ -GATEWAY=@ETH0:GATEWAY@ -HWADDR=@ETH0:HWADDR@ +DEVICE=eth0 +NAME=eth0 +IPADDR={{.NetDevs.eth0.Ipaddr}} +NETMASK={{.NetDevs.eth0.Netmask}} +GATEWAY={{.NetDevs.eth0.Gateway}} +HWADDR={{.NetDevs.eth0.Hwaddr}} BOOTPROTO=static ONBOOT=yes diff --git a/overlays/system/default/etc/sysconfig/network.in b/overlays/system/default/etc/sysconfig/network.in deleted file mode 100644 index 4e0b31ac..00000000 --- a/overlays/system/default/etc/sysconfig/network.in +++ /dev/null @@ -1,2 +0,0 @@ -NETWORKING=yes -HOSTNAME=@FQDN@ diff --git a/overlays/system/default/etc/sysconfig/network.ww b/overlays/system/default/etc/sysconfig/network.ww new file mode 100644 index 00000000..e26cc305 --- /dev/null +++ b/overlays/system/default/etc/sysconfig/network.ww @@ -0,0 +1,2 @@ +NETWORKING=yes +HOSTNAME={{.Fqdn}} diff --git a/overlays/system/default/etc/warewulf/warewulf.conf.in b/overlays/system/default/etc/warewulf/warewulf.conf.in deleted file mode 100644 index f13f49de..00000000 --- a/overlays/system/default/etc/warewulf/warewulf.conf.in +++ /dev/null @@ -1 +0,0 @@ -#WWINCLUDE /etc/warewulf/warewulf.conf diff --git a/overlays/system/default/etc/warewulf/warewulf.conf.ww b/overlays/system/default/etc/warewulf/warewulf.conf.ww new file mode 100644 index 00000000..df0e916e --- /dev/null +++ b/overlays/system/default/etc/warewulf/warewulf.conf.ww @@ -0,0 +1 @@ +{{Include "/etc/warewulf/warewulf.conf"}} diff --git a/overlays/system/default/root/.ssh/authorized_keys b/overlays/system/default/root/.ssh/authorized_keys deleted file mode 100644 index e69de29b..00000000 From 39e8a8c03eacbe9298da643b13960401514555a6 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 00:09:47 -0800 Subject: [PATCH 05/15] Removed unecessary file --- internal/pkg/assets/template.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 internal/pkg/assets/template.go diff --git a/internal/pkg/assets/template.go b/internal/pkg/assets/template.go deleted file mode 100644 index 24f43134..00000000 --- a/internal/pkg/assets/template.go +++ /dev/null @@ -1,21 +0,0 @@ -package assets - - - -type WWTemplate struct { - GroupName string - HostName string - DomainName string - Fqdn string - Vnfs string - VnfsDir string - Ipxe string - SystemOverlay string - RuntimeOverlay string - KernelVersion string - NetDevs map[string]netDevs -} - -func (t WWTemplate) Import(file string) { - -} \ No newline at end of file From 0c3a58e47af8d3d97e759de9fae1c9c3f69e69f1 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 00:34:03 -0800 Subject: [PATCH 06/15] bug fixes --- Makefile | 2 +- cmd/warewulfd/warewulfd.go | 4 ++-- cmd/wwclient/wwclient.go | 2 +- internal/app/wwctl/build/kernel/kernel.go | 2 +- internal/app/wwctl/build/runtime-overlay/runtime-overlay.go | 2 +- internal/app/wwctl/build/system-overlay/system-overlay.go | 2 +- overlays/runtime/default/root/.ssh/authorized_keys.ww | 2 +- .../network-scripts/{ifcfg-eth0.in => ifcfg-eth0.ww} | 0 8 files changed, 8 insertions(+), 8 deletions(-) rename overlays/system/default/etc/sysconfig/network-scripts/{ifcfg-eth0.in => ifcfg-eth0.ww} (100%) diff --git a/Makefile b/Makefile index f792b785..05c0cf58 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ wwclient: clean: rm -f warewulfd - rm -f wwbuild + rm -f wwclient rm -f wwctl install: files services diff --git a/cmd/warewulfd/warewulfd.go b/cmd/warewulfd/warewulfd.go index b9b84552..1cf34ed0 100644 --- a/cmd/warewulfd/warewulfd.go +++ b/cmd/warewulfd/warewulfd.go @@ -66,8 +66,8 @@ func main() { http.HandleFunc("/kernel/", kernelSend) http.HandleFunc("/kmods/", kmodsSend) http.HandleFunc("/vnfs/", vnfsSend) - http.HandleFunc("/system-overlay-system/", systemOverlaySend) - http.HandleFunc("/system-overlay-runtime", runtimeOverlaySend) + http.HandleFunc("/overlay-system/", systemOverlaySend) + http.HandleFunc("/overlay-runtime", runtimeOverlaySend) http.ListenAndServe(":9873", nil) } diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go index 0f71894b..6153639d 100644 --- a/cmd/wwclient/wwclient.go +++ b/cmd/wwclient/wwclient.go @@ -61,7 +61,7 @@ func main() { for true { var err error - getString := fmt.Sprintf("http://%s:%d/system-overlay-runtime", config.Ipaddr, config.Port) + getString := fmt.Sprintf("http://%s:%d/overlay-runtime", config.Ipaddr, config.Port) resp, err = webclient.Get(getString) if err == nil { break diff --git a/internal/app/wwctl/build/kernel/kernel.go b/internal/app/wwctl/build/kernel/kernel.go index 3023264f..3f3670e7 100644 --- a/internal/app/wwctl/build/kernel/kernel.go +++ b/internal/app/wwctl/build/kernel/kernel.go @@ -53,7 +53,7 @@ func Build(nodeList []assets.NodeInfo, force bool) error { if _, err := os.Stat(kernelDrivers); err == nil { if util.PathIsNewer(kernelDrivers, driversDestination) && force == false { - wwlog.Printf(wwlog.INFO, "%-35s: Skipping, kernel Drivers are current\n", "kmods-"+kernelVersion+".img") + wwlog.Printf(wwlog.INFO, "%-35s: Skipping, drivers are current\n", "kmods-"+kernelVersion+".img") } else { cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelDrivers, driversDestination) diff --git a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go index 49be8543..92e9de0b 100644 --- a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go +++ b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go @@ -130,7 +130,7 @@ func Build(nodeList []assets.NodeInfo, force bool) error { wwlog.Printf(wwlog.ERROR, "Could not generate runtime image overlay: %s\n", err) continue } - wwlog.Printf(wwlog.INFO, "%-3s: Done\n", node.Fqdn) + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", node.Fqdn) wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir) os.RemoveAll(tmpDir) diff --git a/internal/app/wwctl/build/system-overlay/system-overlay.go b/internal/app/wwctl/build/system-overlay/system-overlay.go index f4291747..89448be5 100644 --- a/internal/app/wwctl/build/system-overlay/system-overlay.go +++ b/internal/app/wwctl/build/system-overlay/system-overlay.go @@ -87,7 +87,7 @@ func Build(nodeList []assets.NodeInfo, force bool) error { destFile := strings.TrimSuffix(location, ".ww") - tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{"include": fileInclude}).ParseGlob(path.Join(OverlayDir, destFile + ".ww*")) + tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{"Include": fileInclude}).ParseGlob(path.Join(OverlayDir, destFile + ".ww*")) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) return err diff --git a/overlays/runtime/default/root/.ssh/authorized_keys.ww b/overlays/runtime/default/root/.ssh/authorized_keys.ww index 01b8c625..6ec927da 100644 --- a/overlays/runtime/default/root/.ssh/authorized_keys.ww +++ b/overlays/runtime/default/root/.ssh/authorized_keys.ww @@ -1 +1 @@ -{{Include "/root/.ssh/authorized_keys")}} +{{Include "/root/.ssh/authorized_keys"}} diff --git a/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in b/overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww similarity index 100% rename from overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.in rename to overlays/system/default/etc/sysconfig/network-scripts/ifcfg-eth0.ww From fff79227d8c64b9bc313c20b8871663378f10c98 Mon Sep 17 00:00:00 2001 From: Ian Kaneshiro Date: Wed, 11 Nov 2020 10:25:39 -0800 Subject: [PATCH 07/15] Support pulling from docker daemon --- .gitignore | 1 + cmd/wwbuild/wwbuild.go | 12 ++++------ go.sum | 11 +++++++++ internal/app/wwctl/build/vnfs/vnfs.go | 20 +++++++--------- internal/pkg/oci/puller.go | 34 +++++++++++++++++---------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 7334b26a..aad35f33 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor/ warewulfd wwbuild wwclient +wwctl diff --git a/cmd/wwbuild/wwbuild.go b/cmd/wwbuild/wwbuild.go index bc4bcced..ef23ffa9 100644 --- a/cmd/wwbuild/wwbuild.go +++ b/cmd/wwbuild/wwbuild.go @@ -1,21 +1,17 @@ package main -import "os" - -import "os" - - import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "os" "os/exec" "path" "strings" "sync" "time" + + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/vnfs" ) func main() { diff --git a/go.sum b/go.sum index f5191aba..044bc343 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -50,12 +51,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containers/image v1.5.1 h1:ssEuj1c24uJvdMkUa2IrawuEFZBP12p6WzrjNBTQxE0= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= github.com/containers/image/v5 v5.7.0 h1:fiTC8/Xbr+zEP6njGTZtPW/3UD7MC93nC9DbUoWdxkA= github.com/containers/image/v5 v5.7.0/go.mod h1:8aOy+YaItukxghRORkvhq5ibWttHErzDLy6egrKfKos= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= @@ -111,7 +115,9 @@ github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -244,6 +250,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= github.com/mtrmac/gpgme v0.1.2 h1:dNOmvYmsrakgW7LcgiprD0yfRuQQe8/C8F6Z+zogO3s= @@ -494,6 +501,7 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -535,12 +543,14 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -576,6 +586,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/app/wwctl/build/vnfs/vnfs.go b/internal/app/wwctl/build/vnfs/vnfs.go index 1fa9add1..e3f2dc4d 100644 --- a/internal/app/wwctl/build/vnfs/vnfs.go +++ b/internal/app/wwctl/build/vnfs/vnfs.go @@ -1,15 +1,16 @@ package vnfs import ( + "strings" + "github.com/hpcng/warewulf/internal/pkg/assets" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "strings" ) var buildForce bool -func Build(nodeList []assets.NodeInfo, force bool) (error) { +func Build(nodeList []assets.NodeInfo, force bool) error { set := make(map[string]int) wwlog.Printf(wwlog.INFO, "Building and Importing VNFS Images:\n") @@ -19,7 +20,7 @@ func Build(nodeList []assets.NodeInfo, force bool) (error) { for _, node := range nodeList { if node.Vnfs != "" { - set[node.Vnfs] ++ + set[node.Vnfs]++ wwlog.Printf(wwlog.DEBUG, "Node '%s' has VNFS '%s'\n", node.Fqdn, node.Vnfs) } } @@ -27,14 +28,7 @@ func Build(nodeList []assets.NodeInfo, force bool) (error) { for uri := range set { v := vnfs.New(uri) wwlog.Printf(wwlog.VERBOSE, "VNFS found: %s (nodes: %d)\n", uri, set[uri]) - if strings.HasPrefix(uri, "docker://") { - BuildDocker(v) - - } else if strings.HasPrefix(uri, "docker-daemon://") { - //wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service: %s\n", uri) - wwlog.Printf(wwlog.INFO, "Building VNFS from Docker service is not supported yet: %s\n", uri) - - } else if strings.HasPrefix(uri, "/") { + if strings.HasPrefix(uri, "/") { if strings.HasSuffix(uri, "tar.gz") { //wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball: %s\n", uri) wwlog.Printf(wwlog.WARN, "Building VNFS from local tarball is not supported yet: %s\n", uri) @@ -42,10 +36,12 @@ func Build(nodeList []assets.NodeInfo, force bool) (error) { } else { BuildContainerdir(v) } + } else { + BuildDocker(v) } } wwlog.SetIndent(0) return nil -} \ No newline at end of file +} diff --git a/internal/pkg/oci/puller.go b/internal/pkg/oci/puller.go index f04704a9..5c526ebb 100644 --- a/internal/pkg/oci/puller.go +++ b/internal/pkg/oci/puller.go @@ -12,6 +12,7 @@ import ( "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/docker/daemon" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/types" @@ -75,16 +76,28 @@ func stripScheme(uri string) (string, error) { return strings.TrimPrefix(uri, "docker:"), nil } -// generateID stores and returns a unique identifier derived from the sha256sum of the image manifest -func (p *puller) generateID(ctx context.Context, uri string) (string, error) { - cleanURI, err := stripScheme(uri) - if err != nil { - return "", err +// getReference parsed the uri scheme to determine +func getReference(uri string) (types.ImageReference, error) { + s := strings.SplitN(uri, ":", 2) + if len(s) != 2 { + return nil, fmt.Errorf("invalid uri: %q", uri) } - ref, err := docker.ParseReference(cleanURI) + switch s[0] { + case "docker": + return docker.ParseReference(s[1]) + case "docker-daemon": + return daemon.ParseReference(strings.TrimPrefix(s[1], "//")) + default: + return nil, fmt.Errorf("unknown uri sceme: %q", uri) + } +} + +// generateID stores and returns a unique identifier derived from the sha256sum of the image manifest +func (p *puller) generateID(ctx context.Context, uri string) (string, error) { + ref, err := getReference(uri) if err != nil { - return "", err + return "", fmt.Errorf("unable to parse uri: %v", err) } src, err := ref.NewImageSource(ctx, nil) @@ -102,12 +115,7 @@ func (p *puller) generateID(ctx context.Context, uri string) (string, error) { } func (p *puller) pull(ctx context.Context, uri, dst string) (err error) { - cleanURI, err := stripScheme(uri) - if err != nil { - return err - } - - srcRef, err := docker.ParseReference(cleanURI) + srcRef, err := getReference(uri) if err != nil { return fmt.Errorf("unable to parse uri: %v", err) } From 9644690dd540c8c18e2244822498850ef006f5a5 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 15:10:48 -0800 Subject: [PATCH 08/15] Cleanups and removing the legacy wwclient program code --- cmd/wwbuild/overlay-runtime.go | 61 ------------- cmd/wwbuild/overlay-system.go | 59 ------------- cmd/wwbuild/overlay.go | 132 ---------------------------- cmd/wwbuild/wwbuild.go | 151 --------------------------------- cmd/wwclient/wwclient.go | 6 -- internal/pkg/assets/assets.go | 8 +- internal/pkg/config/config.go | 8 +- internal/pkg/vnfs/vnfs.go | 2 +- internal/pkg/wwlog/wwlog.go | 7 +- 9 files changed, 17 insertions(+), 417 deletions(-) delete mode 100644 cmd/wwbuild/overlay-runtime.go delete mode 100644 cmd/wwbuild/overlay-system.go delete mode 100644 cmd/wwbuild/overlay.go delete mode 100644 cmd/wwbuild/wwbuild.go diff --git a/cmd/wwbuild/overlay-runtime.go b/cmd/wwbuild/overlay-runtime.go deleted file mode 100644 index 1b738b01..00000000 --- a/cmd/wwbuild/overlay-runtime.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/util" - "os" - "os/exec" - "path" - "sync" -) - -func overlayRuntime(node assets.NodeInfo, replace map[string]string, wg *sync.WaitGroup) { - defer wg.Done() - - OverlayDir := fmt.Sprintf("%s/overlays/runtime/%s", config.LocalStateDir, node.RuntimeOverlay) - OverlayFile := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", config.LocalStateDir, node.Fqdn) - /* - destModTime := time.Time{} - destMod, err := os.Stat(OverlayFile) - if err == n - - il { - destModTime = destMod.ModTime() - } - configMod, err := os.Stat("/etc/warewulf/nodes.conf") - if err != nil { - fmt.Printf("ERROR: could not find node file: /etc/warewulf/nodes.conf") - os.Exit(1) - } - configModTime := configMod.ModTime() - sourceModTime, _ := util.DirModTime(OverlayDir) - */ - err := os.MkdirAll(path.Dir(OverlayFile), 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - err = os.MkdirAll(OverlayDir, 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - // if sourceModTime.After(destModTime) || configModTime.After(destModTime) { - fmt.Printf("RUNTIME: %s\n", node.Fqdn) - - overlayDest := "/tmp/.system-overlay-" + util.RandomString(16) - BuildOverlayDir(OverlayDir, overlayDest, replace) - - cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) - err = exec.Command("/bin/sh", "-c", cmd).Run() - if err != nil { - fmt.Printf("%s", err) - } - - os.RemoveAll(overlayDest) - // } else { - // fmt.Printf("RUNTIME: %s (skipped no changes)\n", node.Fqdn) - // } -} diff --git a/cmd/wwbuild/overlay-system.go b/cmd/wwbuild/overlay-system.go deleted file mode 100644 index 1fbb0f23..00000000 --- a/cmd/wwbuild/overlay-system.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/util" - "os" - "os/exec" - "path" - "sync" - "time" -) - -func overlaySystem(node assets.NodeInfo, replace map[string]string, wg *sync.WaitGroup) { - defer wg.Done() - - OverlayDir := fmt.Sprintf("%s/overlays/system/%s", config.LocalStateDir, node.SystemOverlay) - OverlayFile := fmt.Sprintf("%s/provision/overlays/system/%s.img", config.LocalStateDir, node.Fqdn) - - destModTime := time.Time{} - destMod, err := os.Stat(OverlayFile) - if err == nil { - destModTime = destMod.ModTime() - } - configMod, err := os.Stat(config.SysConfDir + "nodes.conf") - if err != nil { - fmt.Printf("ERROR: could not find node file: nodes.conf\n", config.SysConfDir) - os.Exit(1) - } - configModTime := configMod.ModTime() - sourceModTime, _ := util.DirModTime(OverlayDir) - - err = os.MkdirAll(path.Dir(OverlayFile), 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - err = os.MkdirAll(OverlayDir, 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - - if sourceModTime.After(destModTime) || configModTime.After(destModTime) { - fmt.Printf("SYSTEM: %s\n", node.Fqdn) - - overlayDest := "/tmp/.system-overlay-" + util.RandomString(16) - BuildOverlayDir(OverlayDir, overlayDest, replace) - - cmd := fmt.Sprintf("cd %s && find . | cpio --quiet -o -H newc -F \"%s\"", overlayDest, OverlayFile) - err = exec.Command("/bin/sh", "-c", cmd).Run() - if err != nil { - fmt.Printf("%s", err) - } - - os.RemoveAll(overlayDest) - } -} diff --git a/cmd/wwbuild/overlay.go b/cmd/wwbuild/overlay.go deleted file mode 100644 index 1ef294e7..00000000 --- a/cmd/wwbuild/overlay.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "github.com/hpcng/warewulf/internal/pkg/util" - "io" - "os" - "path/filepath" - "strings" -) - -func BuildOverlayDir(sourceDir string, destDir string, replace map[string]string) error { - err := os.Chdir(sourceDir) - if err != nil { - fmt.Println(err) - return err - } - - err = filepath.Walk(".", func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - err := os.MkdirAll(destDir+"/"+path, info.Mode()) - if err != nil { - fmt.Println(err) - } - } else { - if filepath.Ext(path) == ".in" { - destFile := strings.TrimSuffix(path, ".in") - skip := false - - sourceFD, err := os.Open(sourceDir + "/" + path) - if err != nil { - return err - } - - destFD, err := os.OpenFile(destDir + "/" + destFile, os.O_RDWR|os.O_CREATE, info.Mode()) - if err != nil { - return err - } - - scanner := bufio.NewScanner(sourceFD) - w := bufio.NewWriter(destFD) - - for scanner.Scan() { - newLine := scanner.Text() - - for k, v := range replace { - replaceString := fmt.Sprintf("@%s@", strings.ToUpper(k)) - newLine = strings.ReplaceAll(newLine, replaceString, v) - } - - if strings.HasPrefix(newLine, "#WWEND") { - skip = false - } else if skip == true { - - } else if strings.HasPrefix(newLine, "#WWIFDEF ") { - line := strings.Split(newLine, " ") - if len(line) > 0 && line[1] != "false" { - skip = true - } - } else if strings.HasPrefix(newLine, "#WWIFNDEF ") { - line := strings.Split(newLine, " ") - if len(line) > 0 && line[1] == "false" { - skip = true - } - } else if strings.HasPrefix(newLine, "#WWIF ") { - line := strings.Split(newLine, " ") - if len(line) == 2 && line[1] != "false" { - skip = true - } else if len(line) > 3 { - - if line[2] == "==" { - if line[1] != line[3] { - skip = true - } - } - } - } else if strings.HasPrefix(newLine, "#WWELSE") { - if skip == true { - skip = false - } else { - skip = true - } - } else if strings.HasPrefix(newLine, "#WWINCLUDE ") { - line := strings.Split(newLine, " ") - //fmt.Printf("Including file (%s): %s\n", destDir + "/" + destFile, line[1]) - includeFD, err := os.Open(line[1]) - if err != nil { - fmt.Printf("ERROR(os.Open): %s\n", err) - return err - } - _, err = io.Copy(w, includeFD) - if err != nil { - fmt.Printf("ERROR(io.Copy): %s\n", err) - return err - } - includeFD.Close() - } else if strings.HasPrefix(newLine, "#WW") { - // Anything else is a comment or to be ignored - continue - } else { - _, err := w.WriteString(newLine + "\n") - if err != nil { - return err - } - } - } - w.Flush() - sourceFD.Close() - destFD.Close() - } else { - err := util.CopyFile(sourceDir+"/"+path, destDir+"/"+path) - - if err != nil { - return err - } - } - } - - return nil - }) - if err != nil { - fmt.Println(err) - return err - } - - return nil -} diff --git a/cmd/wwbuild/wwbuild.go b/cmd/wwbuild/wwbuild.go deleted file mode 100644 index ef23ffa9..00000000 --- a/cmd/wwbuild/wwbuild.go +++ /dev/null @@ -1,151 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/exec" - "path" - "strings" - "sync" - "time" - - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/vnfs" -) - -func main() { - - if len(os.Args) < 2 { - fmt.Printf("USAGE: %s [vnfs/kernel/system-overlay] (node regex)\n", os.Args[0]) - return - } - - if os.Args[1] == "vnfs" { - var nodeList []assets.NodeInfo - set := make(map[string]bool) - var wg sync.WaitGroup - - if len(os.Args) >= 3 { - nodeList, _ = assets.SearchByName(os.Args[2]) - } else { - nodeList, _ = assets.FindAllNodes() - } - - if len(nodeList) == 0 { - fmt.Printf("ERROR: No nodes found\n") - return - } - - for _, node := range nodeList { - if node.Vnfs != "" { - set[node.Vnfs] = true - } - } - - for entry := range set { - wg.Add(1) - if strings.HasPrefix(entry, "/") { - vnfsLocalBuild(entry, &wg) - } else { - vnfsOciBuild(entry, &wg) - } - } - - time.Sleep(1000 * time.Millisecond) - fmt.Printf("Waiting for build(s) to complete...\n") - wg.Wait() - - } else if os.Args[1] == "kernel" { - var nodeList []assets.NodeInfo - set := make(map[string]bool) - - if len(os.Args) >= 3 { - nodeList, _ = assets.SearchByName(os.Args[2]) - } else { - nodeList, _ = assets.FindAllNodes() - } - - if len(nodeList) == 0 { - fmt.Printf("ERROR: No nodes found\n") - return - } - - for _, node := range nodeList { - if node.KernelVersion != "" { - set[node.KernelVersion] = true - } - } - - for kernelVers := range set { - kernelSource := fmt.Sprintf("/boot/vmlinuz-%s", kernelVers) - // TODO: Check time stamps of source and dests to see if we need to rebuild or skip - if _, err := os.Stat(kernelSource); err == nil { - kernelDestination := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", config.LocalStateDir, kernelVers) - kmodsDestination := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", config.LocalStateDir, kernelVers) - - err := os.MkdirAll(path.Dir(kernelDestination), 0755) - if err != nil { - fmt.Printf("ERROR: %s\n", err) - return - } - - fmt.Printf("SETUP KERNEL: %s\n", kernelSource) - err = exec.Command("cp", kernelSource, kernelDestination).Run() - if err != nil { - fmt.Printf("%s", err) - } - - kernelMods := fmt.Sprintf("/lib/modules/%s", kernelVers) - if _, err := os.Stat(kernelMods); err == nil { - fmt.Printf("BUILDING MODS: %s\n", kernelMods) - cmd := fmt.Sprintf("cd /; find .%s | cpio --quiet -o -H newc -F \"%s\"", kernelMods, kmodsDestination) - err := exec.Command("/bin/sh", "-c", cmd).Run() - if err != nil { - fmt.Printf("OUTPUT: %s", err) - } - - } - } - } - } else if os.Args[1] == "system-overlay" { - var nodeList []assets.NodeInfo - var wg sync.WaitGroup - - if len(os.Args) >= 3 { - nodeList, _ = assets.SearchByName(os.Args[2]) - } else { - nodeList, _ = assets.FindAllNodes() - } - - if len(nodeList) == 0 { - fmt.Printf("ERROR: No nodes found\n") - return - } - - for _, node := range nodeList { - v := vnfs.New(node.Vnfs) - replace := make(map[string]string) - replace["HOSTNAME"] = node.HostName - replace["FQDN"] = node.Fqdn - replace["VNFS"] = node.Vnfs - replace["VNFSDIR"] = v.Root() - replace["KERNELVERSION"] = node.KernelVersion - replace["GROUPNAME"] = node.GroupName - replace["DOMAIN"] = node.DomainName - for key, dev := range node.NetDevs { - replace[fmt.Sprintf("%s:NAME", key)] = key - replace[fmt.Sprintf("%s:HWADDR", key)] = strings.ReplaceAll(dev.Hwaddr, "-", ":") - replace[fmt.Sprintf("%s:IPADDR", key)] = dev.Ipaddr - replace[fmt.Sprintf("%s:NETMASK", key)] = dev.Netmask - replace[fmt.Sprintf("%s:GATEWAY", key)] = dev.Gateway - } - - wg.Add(2) - overlayRuntime(node, replace, &wg) - overlaySystem(node, replace, &wg) - } - wg.Wait() - - } -} diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go index 6153639d..9905446c 100644 --- a/cmd/wwclient/wwclient.go +++ b/cmd/wwclient/wwclient.go @@ -78,18 +78,12 @@ func main() { time.Sleep(1000 * time.Millisecond) } -// defer resp.Body.Close() - - - - if resp.StatusCode != 200 { log.Printf("Not updating runtime system-overlay, got status code: %d\n", resp.StatusCode) time.Sleep(60000 * time.Millisecond) continue } - log.Printf("Updating runtime system\n") command := exec.Command("/bin/cpio", "-iu") command.Stdin = resp.Body diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 8415232d..0412d2ff 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -3,6 +3,7 @@ package assets import ( "fmt" "github.com/hpcng/warewulf/internal/pkg/vnfs" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "gopkg.in/yaml.v2" "io/ioutil" "strings" @@ -137,13 +138,16 @@ func FindAllNodes() ([]NodeInfo, error) { n.Fqdn = node.Hostname } - if strings.HasPrefix(n.Vnfs, "docker://") { + if b, _ := regexp.MatchString(`^[a-z\-]+://`, n.Vnfs); b == true { + //if strings.HasPrefix(n.Vnfs, "docker://") { //TODO: This is a kludge and shouldn't be done here. We need to go back // and do this from a "vnfs" interface or package v := vnfs.New(n.Vnfs) n.VnfsDir = v.Root() - } else { + } else if strings.HasPrefix(n.Vnfs, "/"){ n.VnfsDir = n.Vnfs + } else { + wwlog.Printf(wwlog.ERROR, "Configuration 'Vnfs' is invalid for node: %s\n", n.Fqdn) } ret = append(ret, n) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 1717cdd9..acd5236b 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -7,10 +7,10 @@ import ( "io/ioutil" ) -// THIS IS NOT BEING USED YET AND IS THUS A WORK IN PROGRESS - -const SysConfDir = "/etc/warewulf/" -const LocalStateDir = "/var/warewulf" +const ( + SysConfDir = "/etc/warewulf/" + LocalStateDir = "/var/warewulf" +) type Config struct { Port int `yaml:"warewulfd port", envconfig:"WAREWULFD_PORT"` diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 433afcdf..5c39cb00 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -43,7 +43,7 @@ func (self *VnfsObject) NameClean() string { } uri := strings.Split(self.SourcePath, "://") - return strings.ReplaceAll(uri[1], "/", "_") + return strings.ReplaceAll(uri[0]+":"+uri[1], "/", "_") } func (self *VnfsObject) Source() string { diff --git a/internal/pkg/wwlog/wwlog.go b/internal/pkg/wwlog/wwlog.go index 14f36204..085a8c38 100644 --- a/internal/pkg/wwlog/wwlog.go +++ b/internal/pkg/wwlog/wwlog.go @@ -22,7 +22,12 @@ var ( func SetLevel(level uint) { logLevel = level - log.SetFlags(0) + if level == DEBUG { + log.SetFlags(log.Lmicroseconds) + } else { + log.SetFlags(0) + } + Printf(DEBUG, "Set log level to: %d\n", logLevel) } From 9a9672a060f0ad45ca0c124471d52db04b8a3ab7 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 15:11:24 -0800 Subject: [PATCH 09/15] OK, now removing all of the wwclient code! ;-) --- cmd/wwclient/wwclient.go | 97 ---------------------------------------- 1 file changed, 97 deletions(-) delete mode 100644 cmd/wwclient/wwclient.go diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go deleted file mode 100644 index 9905446c..00000000 --- a/cmd/wwclient/wwclient.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "fmt" - "log" - "net" - "net/http" - "os" - "os/exec" - "time" - - "github.com/hpcng/warewulf/internal/pkg/config" -) - -func main() { - if os.Args[0] == "/warewulf/bin/wwclient" { - os.Chdir("/") - log.Printf("Updating live file system LIVE, cancel now if this is in error") - time.Sleep(5000 * time.Millisecond) - } else { - fmt.Printf("Called via: %s\n", os.Args[0]) - fmt.Printf("Runtime system-overlay is being put in '/warewulf/wwclient-test' rather than '/'\n") - os.MkdirAll("/warewulf/wwclient-test", 0755) - os.Chdir("/warewulf/wwclient-test") - } - - config, err := config.New() - if err != nil { - fmt.Printf("ERROR: Could not load configuration file: %s\n", err) - return - } - - localTCPAddr := net.TCPAddr{} - if config.InsecureRuntime == false { - // Setup local port to something privileged (<1024) - localTCPAddr.Port = 987 - } else { - fmt.Printf("INFO: Running from an insecure port\n") - } - - webclient := &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - LocalAddr: &localTCPAddr, - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - for true { - var resp *http.Response - counter := 0 - - for true { - var err error - - getString := fmt.Sprintf("http://%s:%d/overlay-runtime", config.Ipaddr, config.Port) - resp, err = webclient.Get(getString) - if err == nil { - break - } else { - if counter > 60 { - counter = 0 - } - if counter == 0 { - log.Println(err) - } - counter++ - } - - time.Sleep(1000 * time.Millisecond) - } - - if resp.StatusCode != 200 { - log.Printf("Not updating runtime system-overlay, got status code: %d\n", resp.StatusCode) - time.Sleep(60000 * time.Millisecond) - continue - } - - log.Printf("Updating runtime system\n") - command := exec.Command("/bin/cpio", "-iu") - command.Stdin = resp.Body - err := command.Run() - if err != nil { - log.Printf("ERROR: Failed running CPIO: %s\n", err) - } - - time.Sleep(30000 * time.Millisecond) - } -} From fa2a7e26d33ca86a55817b9dfe1f64fd60fb73e2 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 22:16:24 -0800 Subject: [PATCH 10/15] Moving warewulfd code around and implementing new API for config.*() --- cmd/warewulfd/ipxe.go | 69 ------------------ cmd/warewulfd/kernel.go | 35 --------- cmd/warewulfd/kmods.go | 34 --------- cmd/warewulfd/runtime.go | 66 ----------------- cmd/warewulfd/system.go | 34 --------- cmd/warewulfd/vnfs.go | 33 --------- cmd/warewulfd/warewulfd.go | 73 ------------------- internal/app/warewulfd/warewulfd.go | 27 +++++++ internal/app/wwctl/build/kernel/kernel.go | 12 +-- .../build/runtime-overlay/runtime-overlay.go | 6 +- .../build/system-overlay/system-overlay.go | 6 +- internal/app/wwctl/build/vnfs/containerdir.go | 22 ++++-- internal/app/wwctl/build/vnfs/docker.go | 34 ++++++--- internal/app/wwctl/build/vnfs/util.go | 27 ------- internal/pkg/assets/assets.go | 6 +- internal/pkg/config/config.go | 70 ++++++++++++++---- internal/pkg/vnfs/vnfs.go | 23 +----- internal/pkg/wwlog/wwlog.go | 10 +-- 18 files changed, 140 insertions(+), 447 deletions(-) delete mode 100644 cmd/warewulfd/ipxe.go delete mode 100644 cmd/warewulfd/kernel.go delete mode 100644 cmd/warewulfd/kmods.go delete mode 100644 cmd/warewulfd/runtime.go delete mode 100644 cmd/warewulfd/system.go delete mode 100644 cmd/warewulfd/vnfs.go delete mode 100644 cmd/warewulfd/warewulfd.go create mode 100644 internal/app/warewulfd/warewulfd.go diff --git a/cmd/warewulfd/ipxe.go b/cmd/warewulfd/ipxe.go deleted file mode 100644 index e735df80..00000000 --- a/cmd/warewulfd/ipxe.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "log" - "net/http" - "os" - "strconv" - "strings" -) - -func ipxeSend(w http.ResponseWriter, req *http.Request) { - url := strings.Split(req.URL.Path, "/") - - if url[2] == "" { - log.Printf("ERROR: Bad iPXE request from %s\n", req.RemoteAddr) - return - } - - hwaddr := strings.ReplaceAll(url[2], "-", ":") - node, err := assets.FindByHwaddr(hwaddr) - if err != nil { - log.Printf("Could not find HW Addr: %s: %s\n", hwaddr, err) - w.WriteHeader(404) - return - } - - if node.HostName != "" { - log.Printf("IPXE: %15s: %s\n", node.Fqdn, req.URL.Path) - - conf, err := config.New() - if err != nil { - log.Printf("Could not get config: %s\n", err) - return - } - - ipxeTemplate := fmt.Sprintf("/etc/warewulf/ipxe/%s.ipxe", node.Ipxe) - sourceFD, err := os.Open(ipxeTemplate) - if err != nil { - log.Printf("ERROR: Could not open iPXE Template: %s\n", err) - w.WriteHeader(404) - return - } - - scanner := bufio.NewScanner(sourceFD) - - for scanner.Scan() { - newLine := scanner.Text() - - newLine = strings.ReplaceAll(newLine, "@HWADDR@", url[2]) - newLine = strings.ReplaceAll(newLine, "@IPADDR@", conf.Ipaddr) - newLine = strings.ReplaceAll(newLine, "@HOSTNAME@", node.HostName) - newLine = strings.ReplaceAll(newLine, "@FQDN@", node.Fqdn) - newLine = strings.ReplaceAll(newLine, "@PORT@", strconv.Itoa(conf.Port)) - // TODO: Add KernelArgs to nodes.conf - //newLine = strings.ReplaceAll(newLine, "@KERNELARGS@", node.KernelArgs) - - fmt.Fprintln(w, newLine) - } - log.Printf("SEND: %15s: %s\n", node.Fqdn, ipxeTemplate) - - } else { - log.Printf("ERROR: iPXE request from unknown Node (hwaddr=%s)\n", url[2]) - } - return -} diff --git a/cmd/warewulfd/kernel.go b/cmd/warewulfd/kernel.go deleted file mode 100644 index db406f6c..00000000 --- a/cmd/warewulfd/kernel.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" - "log" - "net/http" -) - -func kernelSend(w http.ResponseWriter, req *http.Request) { - - node, err := getSanity(req) - if err != nil { - w.WriteHeader(404) - log.Println(err) - return - } - - if node.KernelVersion != "" { - fileName := fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", config.LocalStateDir, node.KernelVersion) - - err := sendFile(w, fileName, node.Fqdn) - if err != nil { - log.Printf("ERROR: %s\n", err) - } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) - } - - } else { - w.WriteHeader(503) - log.Printf("ERROR: No 'kernel version' set for node %s\n", node.Fqdn) - } - - return -} diff --git a/cmd/warewulfd/kmods.go b/cmd/warewulfd/kmods.go deleted file mode 100644 index 7af8de57..00000000 --- a/cmd/warewulfd/kmods.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" - "log" - "net/http" -) - -func kmodsSend(w http.ResponseWriter, req *http.Request) { - - node, err := getSanity(req) - if err != nil { - w.WriteHeader(404) - log.Println(err) - return - } - if node.KernelVersion != "" { - fileName := fmt.Sprintf("%s/provision/kernel/kmods-%s.img", config.LocalStateDir, node.KernelVersion) - - err := sendFile(w, fileName, node.Fqdn) - if err != nil { - log.Printf("ERROR: %s\n", err) - } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) - } - - } else { - w.WriteHeader(503) - log.Printf("ERROR: No 'kernel version' set for node %s\n", node.Fqdn) - } - - return -} diff --git a/cmd/warewulfd/runtime.go b/cmd/warewulfd/runtime.go deleted file mode 100644 index 4378e3d7..00000000 --- a/cmd/warewulfd/runtime.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/config" - "log" - "net/http" - "strconv" - "strings" -) - -func runtimeOverlaySend(w http.ResponseWriter, req *http.Request) { - remote := strings.Split(req.RemoteAddr, ":") - port, err := strconv.Atoi(remote[1]) - if err != nil { - log.Printf("Could not convert port to integer: %s\n", remote[1]) - w.WriteHeader(503) - return - } - - config, err := config.New() - if err != nil { - fmt.Printf("ERROR: Could not load configuration file: %s\n", err) - return - } - - if config.InsecureRuntime == false { - if port >= 1024 { - log.Panicf("DENIED: Connection coming from non-privledged port: %s\n", req.RemoteAddr) - w.WriteHeader(401) - return - } - } - - node, err := assets.FindByIpaddr(remote[0]) - if err != nil { - fmt.Printf("Could not find node by IP address: %s\n", remote[0]) - w.WriteHeader(404) - return - } - - if node.Fqdn == "" { - log.Printf("UNKNOWN: %15s: %s\n", remote[0], req.URL.Path) - w.WriteHeader(404) - return - } else { - log.Printf("REQ: %15s: %s\n", node.Fqdn, req.URL.Path) - } - - if node.RuntimeOverlay != "" { - fileName := fmt.Sprintf("%s/provision/overlays/runtime/%s.img", config.LocalStateDir, node.Fqdn) - - err := sendFile(w, fileName, node.Fqdn) - if err != nil { - log.Printf("ERROR: %s\n", err) - } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) - } - } else { - w.WriteHeader(503) - log.Printf("ERROR: No 'runtime system-overlay' set for node %s\n", node.Fqdn) - } - - return -} diff --git a/cmd/warewulfd/system.go b/cmd/warewulfd/system.go deleted file mode 100644 index 3007fdf8..00000000 --- a/cmd/warewulfd/system.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "fmt" - "github.com/hpcng/warewulf/internal/pkg/config" - "log" - "net/http" -) - -func systemOverlaySend(w http.ResponseWriter, req *http.Request) { - - node, err := getSanity(req) - if err != nil { - w.WriteHeader(404) - log.Println(err) - return - } - - if node.SystemOverlay != "" { - fileName := fmt.Sprintf("%s/provision/overlays/system/%s.img", config.LocalStateDir, node.Fqdn) - - err := sendFile(w, fileName, node.Fqdn) - if err != nil { - log.Printf("ERROR: %s\n", err) - } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) - } - } else { - w.WriteHeader(503) - log.Printf("ERROR: No 'system system-overlay' set for node %s\n", node.Fqdn) - } - - return -} diff --git a/cmd/warewulfd/vnfs.go b/cmd/warewulfd/vnfs.go deleted file mode 100644 index 675a3ffc..00000000 --- a/cmd/warewulfd/vnfs.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "log" - "net/http" -) - -func vnfsSend(w http.ResponseWriter, req *http.Request) { - - node, err := getSanity(req) - if err != nil { - w.WriteHeader(404) - log.Panicln(err) - return - } - - if node.Vnfs != "" { - v := vnfs.New(node.Vnfs) - - err := sendFile(w, v.Image(), node.Fqdn) - if err != nil { - log.Printf("ERROR: %s\n", err) - } else { - log.Printf("SEND: %15s: %s\n", node.Fqdn, v.Image()) - } - } else { - w.WriteHeader(503) - log.Printf("ERROR: No VNFS set for node %s\n", node.Fqdn) - } - - return -} diff --git a/cmd/warewulfd/warewulfd.go b/cmd/warewulfd/warewulfd.go deleted file mode 100644 index 1cf34ed0..00000000 --- a/cmd/warewulfd/warewulfd.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "io" - "log" - "os" - "strconv" - "strings" - - "github.com/hpcng/warewulf/internal/pkg/assets" - "github.com/hpcng/warewulf/internal/pkg/errors" - "net/http" -) - -// TODO: https://github.com/danderson/netboot/blob/master/pixiecore/dhcp.go -// TODO: https://github.com/pin/tftp - -//const LocalStateDir = "/var/warewulf" - -func getSanity(req *http.Request) (assets.NodeInfo, error) { - url := strings.Split(req.URL.Path, "/") - - hwaddr := strings.ReplaceAll(url[2], "-", ":") - node, err := assets.FindByHwaddr(hwaddr) - if err != nil { - return node, errors.New("Could not find node by HW address") - } - - if node.Fqdn == "" { - log.Printf("UNKNOWN: %15s: %s\n", hwaddr, req.URL.Path) - return node, errors.New("Unknown node HW address: " + hwaddr) - } else { - log.Printf("REQ: %15s: %s\n", node.Fqdn, req.URL.Path) - } - - return node, nil -} - -func sendFile(w http.ResponseWriter, filename string, sendto string) error { - - fd, err := os.Open(filename) - if err != nil { - return err - } - - FileHeader := make([]byte, 512) - fd.Read(FileHeader) - FileContentType := http.DetectContentType(FileHeader) - FileStat, _ := fd.Stat() - FileSize := strconv.FormatInt(FileStat.Size(), 10) - - w.Header().Set("Content-Disposition", "attachment; filename=kernel") - w.Header().Set("Content-Type", FileContentType) - w.Header().Set("Content-Length", FileSize) - - fd.Seek(0, 0) - io.Copy(w, fd) - - fd.Close() - return nil -} - -func main() { - - http.HandleFunc("/ipxe/", ipxeSend) - http.HandleFunc("/kernel/", kernelSend) - http.HandleFunc("/kmods/", kmodsSend) - http.HandleFunc("/vnfs/", vnfsSend) - http.HandleFunc("/overlay-system/", systemOverlaySend) - http.HandleFunc("/overlay-runtime", runtimeOverlaySend) - - http.ListenAndServe(":9873", nil) -} diff --git a/internal/app/warewulfd/warewulfd.go b/internal/app/warewulfd/warewulfd.go new file mode 100644 index 00000000..57ec3281 --- /dev/null +++ b/internal/app/warewulfd/warewulfd.go @@ -0,0 +1,27 @@ +package warewulfd + +import ( + warewulfd_responses "github.com/hpcng/warewulf/internal/app/warewulfd/warewulfd-reponses" + "github.com/spf13/cobra" + "net/http" +) + +// TODO: https://github.com/danderson/netboot/blob/master/pixiecore/dhcp.go +// TODO: https://github.com/pin/tftp + +//const LocalStateDir = "/var/warewulf" + + +func CobraRunE(cmd *cobra.Command, args []string) error { + + http.HandleFunc("/ipxe/", warewulfd_responses.IpxeSend) + http.HandleFunc("/kernel/", warewulfd_responses.KernelSend) + http.HandleFunc("/kmods/", warewulfd_responses.KmodsSend) + http.HandleFunc("/vnfs/", warewulfd_responses.VnfsSend) + http.HandleFunc("/overlay-system/", warewulfd_responses.SystemOverlaySend) + http.HandleFunc("/overlay-runtime", warewulfd_responses.RuntimeOverlaySend) + + http.ListenAndServe(":9873", nil) + + return nil +} diff --git a/internal/app/wwctl/build/kernel/kernel.go b/internal/app/wwctl/build/kernel/kernel.go index 3f3670e7..0a68f278 100644 --- a/internal/app/wwctl/build/kernel/kernel.go +++ b/internal/app/wwctl/build/kernel/kernel.go @@ -11,9 +11,8 @@ import ( "path" ) -const kernelProvisionPath = "/provision/kernel/" - func Build(nodeList []assets.NodeInfo, force bool) error { + config := config.New() set := make(map[string]int) wwlog.Printf(wwlog.INFO, "Importing Kernels:\n") @@ -29,12 +28,13 @@ func Build(nodeList []assets.NodeInfo, force bool) error { for kernelVersion := range set { kernelImage := "/boot/vmlinuz-"+kernelVersion kernelDrivers := "/lib/modules/"+kernelVersion - kernelDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "vmlinuz-"+kernelVersion) - driversDestination := path.Join(config.LocalStateDir, kernelProvisionPath, "kmods-"+kernelVersion+".img") + kernelDestination := config.KernelImage(kernelVersion) + driversDestination := config.KmodsImage(kernelVersion) + // Create the destination paths just in case it doesn't exist + os.MkdirAll(path.Dir(kernelDestination), 0755) + os.MkdirAll(path.Dir(driversDestination), 0755) - // Create the kernel destination path just in case it doesn't exist - os.MkdirAll(path.Join(config.LocalStateDir, kernelProvisionPath), 0755) if _, err := os.Stat(kernelImage); err == nil { if util.PathIsNewer(kernelImage, kernelDestination) && force == false { diff --git a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go index 92e9de0b..b1168aa1 100644 --- a/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go +++ b/internal/app/wwctl/build/runtime-overlay/runtime-overlay.go @@ -27,12 +27,14 @@ func fileInclude(path string) string { func Build(nodeList []assets.NodeInfo, force bool) error { wwlog.Printf(wwlog.INFO, "Building Runtime Overlays:\n") + config := config.New() + wwlog.SetIndent(4) for _, node := range nodeList { if node.RuntimeOverlay != "" { - OverlayDir := path.Join(config.LocalStateDir, "/overlays/runtime/", node.RuntimeOverlay) - OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/runtime/", node.Fqdn+".img") + OverlayDir := config.RuntimeOverlaySource(node.RuntimeOverlay) + OverlayFile := config.RuntimeOverlayImage(node.Fqdn) wwlog.Printf(wwlog.VERBOSE, "Building Runtime Overlay for: %s\n", node.Fqdn) diff --git a/internal/app/wwctl/build/system-overlay/system-overlay.go b/internal/app/wwctl/build/system-overlay/system-overlay.go index 89448be5..2e1dee02 100644 --- a/internal/app/wwctl/build/system-overlay/system-overlay.go +++ b/internal/app/wwctl/build/system-overlay/system-overlay.go @@ -27,12 +27,14 @@ func fileInclude(path string) string { func Build(nodeList []assets.NodeInfo, force bool) error { wwlog.Printf(wwlog.INFO, "Building System Overlays:\n") + config := config.New() + wwlog.SetIndent(4) for _, node := range nodeList { if node.SystemOverlay != "" { - OverlayDir := path.Join(config.LocalStateDir, "/overlays/system/", node.SystemOverlay) - OverlayFile := path.Join(config.LocalStateDir, "/provision/overlays/system/", node.Fqdn+".img") + OverlayDir := config.SystemOverlaySource(node.SystemOverlay) + OverlayFile := config.SystemOverlayImage(node.Fqdn) wwlog.Printf(wwlog.VERBOSE, "Building System Overlay for: %s\n", node.Fqdn) diff --git a/internal/app/wwctl/build/vnfs/containerdir.go b/internal/app/wwctl/build/vnfs/containerdir.go index b3cb6be4..c3725219 100644 --- a/internal/app/wwctl/build/vnfs/containerdir.go +++ b/internal/app/wwctl/build/vnfs/containerdir.go @@ -2,6 +2,7 @@ package vnfs import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" @@ -10,6 +11,7 @@ import ( ) func BuildContainerdir(v vnfs.VnfsObject) { + config := config.New() if _, err := os.Stat(v.Source()); err != nil { wwlog.Printf(wwlog.INFO, "%-35s: Skipping (bad path)\n", v.Name()) @@ -17,31 +19,35 @@ func BuildContainerdir(v vnfs.VnfsObject) { } wwlog.Printf(wwlog.DEBUG, "Checking if there have been any updates to the VNFS directory\n") - if util.PathIsNewer(v.Source(), v.Image()) { + if util.PathIsNewer(v.Source(), config.VnfsImage(v.NameClean())) { if buildForce == false { wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name()) return } } - wwlog.Printf(wwlog.DEBUG, "Making the directory: %s\n", path.Dir(v.Image())) - err := os.MkdirAll(path.Dir(v.Image()), 0755) + wwlog.Printf(wwlog.DEBUG, "Making the directory: %s\n", path.Dir(config.VnfsImage(v.NameClean()))) + err := os.MkdirAll(path.Dir(config.VnfsImage(v.NameClean())), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", v.Source(), v.Image()) - err = buildVnfs(v.Source(), v.Image()) + wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", v.Source(), config.VnfsImage(v.NameClean())) + err = buildVnfs(v.Source(), config.VnfsImage(v.NameClean())) if err != nil { fmt.Printf("ERROR: %s\n", err) os.Exit(1) } - wwlog.Printf(wwlog.DEBUG, "Building links for Warewulf access to chroot\n") - err = buildLinks(v, v.Source()) + // Setup links from OCI rootfs to chroot path + _ = os.Remove(config.VnfsChroot(v.NameClean()) + "-link") + err = os.Symlink(v.Source(), config.VnfsChroot(v.NameClean())+"-link") + if err != nil { + os.Exit(1) + } + err = os.Rename(config.VnfsChroot(v.NameClean())+"-link", config.VnfsChroot(v.NameClean())) if err != nil { - fmt.Printf("ERROR: %s\n", err) os.Exit(1) } diff --git a/internal/app/wwctl/build/vnfs/docker.go b/internal/app/wwctl/build/vnfs/docker.go index 79ea8825..061e4989 100644 --- a/internal/app/wwctl/build/vnfs/docker.go +++ b/internal/app/wwctl/build/vnfs/docker.go @@ -11,13 +11,13 @@ import ( "path" ) -const ( - OciCacheDir = config.LocalStateDir + "/oci" - VnfsHashDir = config.LocalStateDir + "/oci/vnfs/hash/" -) func BuildDocker(v vnfs.VnfsObject) { wwlog.Printf(wwlog.VERBOSE, "Building OCI Container: %s\n", v.Source()) + config := config.New() + + OciCacheDir := config.LocalStateDir + "/oci" + VnfsHashDir := config.LocalStateDir + "/oci/vnfs" c, err := oci.NewCache(oci.OptSetCachePath(OciCacheDir)) if err != nil { @@ -34,7 +34,7 @@ func BuildDocker(v vnfs.VnfsObject) { hashDestination := VnfsHashDir + path.Base(sourcePath) - name, err := os.Readlink(v.Image()) + name, err := os.Readlink(config.VnfsImage(v.NameClean())) if err == nil { if name == hashDestination { wwlog.Printf(wwlog.INFO, "%-35s: Skipping, VNFS is current\n", v.Name()) @@ -47,12 +47,12 @@ func BuildDocker(v vnfs.VnfsObject) { fmt.Printf("ERROR: %s\n", err) return } - err = os.MkdirAll(path.Dir(v.Image()), 0755) + err = os.MkdirAll(path.Dir(config.VnfsImage(v.NameClean())), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return } - err = os.MkdirAll(path.Dir(v.Root()), 0755) + err = os.MkdirAll(path.Dir(config.VnfsChroot(v.NameClean())), 0755) if err != nil { fmt.Printf("ERROR: %s\n", err) return @@ -68,19 +68,29 @@ func BuildDocker(v vnfs.VnfsObject) { wwlog.Printf(wwlog.VERBOSE, "Finalizing Build\n") - _ = os.Remove(v.Image() + "-link") - err = os.Symlink(hashDestination, v.Image()+"-link") + // Setup links from OCI image to provision path + _ = os.Remove(config.VnfsImage(v.NameClean()) + "-link") + err = os.Symlink(hashDestination, config.VnfsImage(v.NameClean())+"-link") if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } - err = os.Rename(v.Image()+"-link", v.Image()) + err = os.Rename(config.VnfsImage(v.NameClean())+"-link", config.VnfsImage(v.NameClean())) + if err != nil { + os.Exit(1) + } - err = buildLinks(v, sourcePath) + // Setup links from OCI rootfs to chroot path + _ = os.Remove(config.VnfsChroot(v.NameClean()) + "-link") + err = os.Symlink(sourcePath, config.VnfsChroot(v.NameClean())+"-link") if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } + err = os.Rename(config.VnfsChroot(v.NameClean())+"-link", config.VnfsChroot(v.NameClean())) + if err != nil { + os.Exit(1) + } + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", v.Name()) diff --git a/internal/app/wwctl/build/vnfs/util.go b/internal/app/wwctl/build/vnfs/util.go index 8bd44e3d..91d26a39 100644 --- a/internal/app/wwctl/build/vnfs/util.go +++ b/internal/app/wwctl/build/vnfs/util.go @@ -1,12 +1,8 @@ package vnfs - import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "os" "os/exec" - "path" ) func buildVnfs(source string, dest string) error { @@ -16,26 +12,3 @@ func buildVnfs(source string, dest string) error { return err } - -func buildLinks(v vnfs.VnfsObject, source string) error { - - // Just incase the temporary link location is present, remove it if we can - _ = os.Remove(v.Root() + "-link") - - // Just incase the directory doesn't exist, make it - _ = os.MkdirAll(path.Dir(v.Root()), 0755) - - // Link to a temporary location so we can atomically move the link into place - err := os.Symlink(source, v.Root()+"-link") - if err != nil { - return err - } - - // Atomically move the link into place - err = os.Rename(v.Root()+"-link", v.Root()) - if err != nil { - return err - } - - return nil -} diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 0412d2ff..025383e5 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -2,12 +2,8 @@ package assets import ( "fmt" - "github.com/hpcng/warewulf/internal/pkg/vnfs" - "github.com/hpcng/warewulf/internal/pkg/wwlog" "gopkg.in/yaml.v2" "io/ioutil" - "strings" - // "os" // "os" @@ -138,6 +134,7 @@ func FindAllNodes() ([]NodeInfo, error) { n.Fqdn = node.Hostname } + /* if b, _ := regexp.MatchString(`^[a-z\-]+://`, n.Vnfs); b == true { //if strings.HasPrefix(n.Vnfs, "docker://") { //TODO: This is a kludge and shouldn't be done here. We need to go back @@ -149,6 +146,7 @@ func FindAllNodes() ([]NodeInfo, error) { } else { wwlog.Printf(wwlog.ERROR, "Configuration 'Vnfs' is invalid for node: %s\n", n.Fqdn) } + */ ret = append(ret, n) } diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index acd5236b..6b759e05 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -2,15 +2,13 @@ package config import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/kelseyhightower/envconfig" "gopkg.in/yaml.v2" "io/ioutil" + "os" ) -const ( - SysConfDir = "/etc/warewulf/" - LocalStateDir = "/var/warewulf" -) type Config struct { Port int `yaml:"warewulfd port", envconfig:"WAREWULFD_PORT"` @@ -21,34 +19,76 @@ type Config struct { LocalStateDir string `yaml:"local state dir"` } -func New() (Config, error) { - var c Config +var c Config - fd, err := ioutil.ReadFile(SysConfDir + "warewulf.conf") +func init() { + fd, err := ioutil.ReadFile("/etc/warewulf/warewulf.conf") if err != nil { - return c, err + wwlog.Printf(wwlog.ERROR, "Could not read config file: %s\n", err) + os.Exit(255) } err = yaml.Unmarshal(fd, &c) if err != nil { - return c, err + wwlog.Printf(wwlog.ERROR, "Could not unmarshal config file: %s\n", err) + os.Exit(255) } err = envconfig.Process("", &c) if err != nil { - return c, err + wwlog.Printf(wwlog.ERROR, "Could not obtain environment configuration: %s\n", err) + os.Exit(255) } if c.Ipaddr == "" { - fmt.Printf("ERROR: 'warewulf ipaddr' has not been set in %s\n", SysConfDir+"warewulf.conf") + fmt.Printf("ERROR: 'warewulf ipaddr' has not been set in /etc/warewulf/warewulf.conf\n") } if c.SysConfDir == "" { - c.SysConfDir = SysConfDir + c.SysConfDir = "/etc/warewulf" } if c.LocalStateDir == "" { - c.LocalStateDir = LocalStateDir + c.LocalStateDir = "/var/warewulf" } - - return c, nil } + +func New() (Config) { + return c +} + +func (self *Config) NodeConfig() (string) { + return fmt.Sprintf("%s/nodes.conf", self.LocalStateDir) +} + +func (self *Config) SystemOverlaySource(overlayName string) (string) { + return fmt.Sprintf("%s/overlays/system/%s", self.LocalStateDir, overlayName) +} + +func (self *Config) RuntimeOverlaySource(overlayName string) (string) { + return fmt.Sprintf("%s/overlays/runtime/%s", self.LocalStateDir, overlayName) +} + +func (self *Config) KernelImage(kernelVersion string) (string) { + return fmt.Sprintf("%s/kernel/vmlinuz-%s", self.LocalStateDir, kernelVersion) +} + +func (self *Config) KmodsImage(kernelVersion string) (string) { + return fmt.Sprintf("%s/kernel/kmods-%s.img", self.LocalStateDir, kernelVersion) +} + +func (self *Config) VnfsImage(vnfsNameClean string) (string) { + return fmt.Sprintf("%s/vnfs/%s.img.gz", self.LocalStateDir, vnfsNameClean) +} + +func (self *Config) SystemOverlayImage(nodeName string) (string) { + return fmt.Sprintf("%s/overlay/system/%s.img", self.LocalStateDir, nodeName) +} + +func (self *Config) RuntimeOverlayImage(nodeName string) (string) { + return fmt.Sprintf("%s/overlay/runtime/%s.img", self.LocalStateDir, nodeName) +} + +func (self *Config) VnfsChroot(vnfsNameClean string) (string) { + return fmt.Sprintf("%s/chroot/%s.img", self.LocalStateDir, vnfsNameClean) +} + diff --git a/internal/pkg/vnfs/vnfs.go b/internal/pkg/vnfs/vnfs.go index 5c39cb00..aaf360a5 100644 --- a/internal/pkg/vnfs/vnfs.go +++ b/internal/pkg/vnfs/vnfs.go @@ -1,15 +1,12 @@ package vnfs import ( - "github.com/hpcng/warewulf/internal/pkg/config" "path" "strings" ) type VnfsObject struct { SourcePath string - RootPath string - ImagePath string } func New(s string) VnfsObject { @@ -20,7 +17,6 @@ func New(s string) VnfsObject { return ret } - func (self *VnfsObject) Name() string { if self.SourcePath == "" { return "" @@ -52,21 +48,4 @@ func (self *VnfsObject) Source() string { } return self.SourcePath -} - - -func (self *VnfsObject) Image() string { - if self.SourcePath == "" { - return "" - } - - return config.LocalStateDir + "/provision/vnfs/" + self.NameClean() + ".img.gz" -} - -func (self *VnfsObject) Root() string { - if self.SourcePath == "" { - return "" - } - - return config.LocalStateDir + "/chroots/" + self.NameClean() -} +} \ No newline at end of file diff --git a/internal/pkg/wwlog/wwlog.go b/internal/pkg/wwlog/wwlog.go index 085a8c38..1b240d7f 100644 --- a/internal/pkg/wwlog/wwlog.go +++ b/internal/pkg/wwlog/wwlog.go @@ -15,11 +15,11 @@ const ( ) var ( - logLevel uint + logLevel = INFO Indent string ) -func SetLevel(level uint) { +func SetLevel(level int) { logLevel = level if level == DEBUG { @@ -35,7 +35,7 @@ func SetIndent(i int) { Indent = strings.Repeat(" ", i) } -func prefixLevel(level uint) { +func prefixLevel(level int) { if level == DEBUG { log.SetPrefix("[DEBUG] "+Indent) } else if level == VERBOSE { @@ -51,7 +51,7 @@ func prefixLevel(level uint) { } } -func Println(level uint, message string) { +func Println(level int, message string) { if level <= logLevel { prefixLevel(level) log.Println(message) @@ -60,7 +60,7 @@ func Println(level uint, message string) { log.SetPrefix("[LOG] "+Indent) } -func Printf(level uint, message string, a...interface{}) { +func Printf(level int, message string, a...interface{}) { if level <= logLevel { prefixLevel(level) log.Printf(message, a...) From 73c93c362fc7a9f73d1df7b3303a162fa8a104e1 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 22:18:58 -0800 Subject: [PATCH 11/15] Fixing oops of deletion of wwclient. I meant to delete old wwctl bits. --- cmd/wwclient/wwclient.go | 103 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 cmd/wwclient/wwclient.go diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go new file mode 100644 index 00000000..6153639d --- /dev/null +++ b/cmd/wwclient/wwclient.go @@ -0,0 +1,103 @@ +package main + +import ( + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "time" + + "github.com/hpcng/warewulf/internal/pkg/config" +) + +func main() { + if os.Args[0] == "/warewulf/bin/wwclient" { + os.Chdir("/") + log.Printf("Updating live file system LIVE, cancel now if this is in error") + time.Sleep(5000 * time.Millisecond) + } else { + fmt.Printf("Called via: %s\n", os.Args[0]) + fmt.Printf("Runtime system-overlay is being put in '/warewulf/wwclient-test' rather than '/'\n") + os.MkdirAll("/warewulf/wwclient-test", 0755) + os.Chdir("/warewulf/wwclient-test") + } + + config, err := config.New() + if err != nil { + fmt.Printf("ERROR: Could not load configuration file: %s\n", err) + return + } + + localTCPAddr := net.TCPAddr{} + if config.InsecureRuntime == false { + // Setup local port to something privileged (<1024) + localTCPAddr.Port = 987 + } else { + fmt.Printf("INFO: Running from an insecure port\n") + } + + webclient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + LocalAddr: &localTCPAddr, + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } + + for true { + var resp *http.Response + counter := 0 + + for true { + var err error + + getString := fmt.Sprintf("http://%s:%d/overlay-runtime", config.Ipaddr, config.Port) + resp, err = webclient.Get(getString) + if err == nil { + break + } else { + if counter > 60 { + counter = 0 + } + if counter == 0 { + log.Println(err) + } + counter++ + } + + time.Sleep(1000 * time.Millisecond) + } + +// defer resp.Body.Close() + + + + + if resp.StatusCode != 200 { + log.Printf("Not updating runtime system-overlay, got status code: %d\n", resp.StatusCode) + time.Sleep(60000 * time.Millisecond) + continue + } + + + log.Printf("Updating runtime system\n") + command := exec.Command("/bin/cpio", "-iu") + command.Stdin = resp.Body + err := command.Run() + if err != nil { + log.Printf("ERROR: Failed running CPIO: %s\n", err) + } + + time.Sleep(30000 * time.Millisecond) + } +} From aeec20fa21765d8d4966b82481692c391e28ea63 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 22:21:55 -0800 Subject: [PATCH 12/15] Minor typo fix for wwclient. --- cmd/wwclient/wwclient.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/wwclient/wwclient.go b/cmd/wwclient/wwclient.go index 6153639d..cdac4a9b 100644 --- a/cmd/wwclient/wwclient.go +++ b/cmd/wwclient/wwclient.go @@ -24,11 +24,7 @@ func main() { os.Chdir("/warewulf/wwclient-test") } - config, err := config.New() - if err != nil { - fmt.Printf("ERROR: Could not load configuration file: %s\n", err) - return - } + config := config.New() localTCPAddr := net.TCPAddr{} if config.InsecureRuntime == false { From 46bde8b54d92ff9823afbcb49dadb52b55993e1a Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 22:33:13 -0800 Subject: [PATCH 13/15] Something very weird happened with git for me, here is the fix. --- cmd/warewulfd/main.go | 12 ++++ internal/app/warewulfd/root.go | 40 +++++++++++ .../app/warewulfd/warewulfd-reponses/ipxe.go | 66 ++++++++++++++++++ .../warewulfd/warewulfd-reponses/kernel.go | 35 ++++++++++ .../app/warewulfd/warewulfd-reponses/kmods.go | 34 ++++++++++ .../warewulfd/warewulfd-reponses/runtime.go | 67 +++++++++++++++++++ .../warewulfd/warewulfd-reponses/system.go | 34 ++++++++++ .../app/warewulfd/warewulfd-reponses/util.go | 55 +++++++++++++++ .../app/warewulfd/warewulfd-reponses/vnfs.go | 35 ++++++++++ 9 files changed, 378 insertions(+) create mode 100644 cmd/warewulfd/main.go create mode 100644 internal/app/warewulfd/root.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/ipxe.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/kernel.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/kmods.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/runtime.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/system.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/util.go create mode 100644 internal/app/warewulfd/warewulfd-reponses/vnfs.go diff --git a/cmd/warewulfd/main.go b/cmd/warewulfd/main.go new file mode 100644 index 00000000..019de38f --- /dev/null +++ b/cmd/warewulfd/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/hpcng/warewulf/internal/app/warewulfd" +) + + +func main() { + root := warewulfd.GetRootCommand() + + root.Execute() +} \ No newline at end of file diff --git a/internal/app/warewulfd/root.go b/internal/app/warewulfd/root.go new file mode 100644 index 00000000..a4dd41a1 --- /dev/null +++ b/internal/app/warewulfd/root.go @@ -0,0 +1,40 @@ +package warewulfd + +import ( + "github.com/hpcng/warewulf/internal/pkg/wwlog" + "github.com/spf13/cobra" +) + +var ( + WarewulfdCmd = &cobra.Command{ + Use: "warewulfd", + Short: "Warewulf Daemon Service", + Long: "This is the primary Warewulf service for provisioning nodes", + RunE: CobraRunE, + PersistentPreRunE: rootPersistentPreRunE, + } + verboseArg bool + debugArg bool +) + + +func init() { + WarewulfdCmd.PersistentFlags().BoolVarP(&verboseArg, "verbose", "v", false, "Run with increased verbosity.") + WarewulfdCmd.PersistentFlags().BoolVarP(&debugArg, "debug", "d", false, "Run with debugging messages enabled.") +} + +// GetRootCommand returns the root cobra.Command for the application. +func GetRootCommand() *cobra.Command { + return WarewulfdCmd +} + +func rootPersistentPreRunE(cmd *cobra.Command, args []string) error { + if verboseArg == true { + wwlog.SetLevel(wwlog.VERBOSE) + } else if debugArg == true { + wwlog.SetLevel(wwlog.DEBUG) + } else { + wwlog.SetLevel(wwlog.INFO) + } + return nil +} \ No newline at end of file diff --git a/internal/app/warewulfd/warewulfd-reponses/ipxe.go b/internal/app/warewulfd/warewulfd-reponses/ipxe.go new file mode 100644 index 00000000..0cb2b262 --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/ipxe.go @@ -0,0 +1,66 @@ +package warewulfd_responses + +import ( + "bufio" + "fmt" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "log" + "net/http" + "os" + "strconv" + "strings" +) + +func IpxeSend(w http.ResponseWriter, req *http.Request) { + url := strings.Split(req.URL.Path, "/") + + if url[2] == "" { + log.Printf("ERROR: Bad iPXE request from %s\n", req.RemoteAddr) + return + } + + hwaddr := strings.ReplaceAll(url[2], "-", ":") + node, err := assets.FindByHwaddr(hwaddr) + if err != nil { + log.Printf("Could not find HW Addr: %s: %s\n", hwaddr, err) + w.WriteHeader(404) + return + } + + if node.HostName != "" { + log.Printf("IPXE: %15s: %s\n", node.Fqdn, req.URL.Path) + + conf := config.New() + + ipxeTemplate := fmt.Sprintf("/etc/warewulf/ipxe/%s.ipxe", node.Ipxe) + sourceFD, err := os.Open(ipxeTemplate) + if err != nil { + log.Printf("ERROR: Could not open iPXE Template: %s\n", err) + w.WriteHeader(404) + return + } + + scanner := bufio.NewScanner(sourceFD) + + for scanner.Scan() { + newLine := scanner.Text() + + // TODO: Update this to use templates instead of replaces + newLine = strings.ReplaceAll(newLine, "@HWADDR@", url[2]) + newLine = strings.ReplaceAll(newLine, "@IPADDR@", conf.Ipaddr) + newLine = strings.ReplaceAll(newLine, "@HOSTNAME@", node.HostName) + newLine = strings.ReplaceAll(newLine, "@FQDN@", node.Fqdn) + newLine = strings.ReplaceAll(newLine, "@PORT@", strconv.Itoa(conf.Port)) + // TODO: Add KernelArgs to nodes.conf + //newLine = strings.ReplaceAll(newLine, "@KERNELARGS@", node.KernelArgs) + + fmt.Fprintln(w, newLine) + } + log.Printf("SEND: %15s: %s\n", node.Fqdn, ipxeTemplate) + + } else { + log.Printf("ERROR: iPXE request from unknown Node (hwaddr=%s)\n", url[2]) + } + return +} diff --git a/internal/app/warewulfd/warewulfd-reponses/kernel.go b/internal/app/warewulfd/warewulfd-reponses/kernel.go new file mode 100644 index 00000000..018a9d8f --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/kernel.go @@ -0,0 +1,35 @@ +package warewulfd_responses + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "log" + "net/http" +) + +func KernelSend(w http.ResponseWriter, req *http.Request) { + config := config.New() + + node, err := getSanity(req) + if err != nil { + w.WriteHeader(404) + log.Println(err) + return + } + + if node.KernelVersion != "" { + fileName := config.KernelImage(node.KernelVersion) + + err := sendFile(w, fileName, node.Fqdn) + if err != nil { + log.Printf("ERROR: %s\n", err) + } else { + log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) + } + + } else { + w.WriteHeader(503) + log.Printf("ERROR: No 'kernel version' set for node %s\n", node.Fqdn) + } + + return +} diff --git a/internal/app/warewulfd/warewulfd-reponses/kmods.go b/internal/app/warewulfd/warewulfd-reponses/kmods.go new file mode 100644 index 00000000..dff142f1 --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/kmods.go @@ -0,0 +1,34 @@ +package warewulfd_responses + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "log" + "net/http" +) + +func KmodsSend(w http.ResponseWriter, req *http.Request) { + config := config.New() + + node, err := getSanity(req) + if err != nil { + w.WriteHeader(404) + log.Println(err) + return + } + if node.KernelVersion != "" { + fileName := config.KmodsImage(node.KernelVersion) + + err := sendFile(w, fileName, node.Fqdn) + if err != nil { + log.Printf("ERROR: %s\n", err) + } else { + log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) + } + + } else { + w.WriteHeader(503) + log.Printf("ERROR: No 'kernel version' set for node %s\n", node.Fqdn) + } + + return +} diff --git a/internal/app/warewulfd/warewulfd-reponses/runtime.go b/internal/app/warewulfd/warewulfd-reponses/runtime.go new file mode 100644 index 00000000..b173ddc7 --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/runtime.go @@ -0,0 +1,67 @@ +package warewulfd_responses + +import ( + "fmt" + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/config" + "log" + "net/http" + "strconv" + "strings" +) + +func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { + config := config.New() + + remote := strings.Split(req.RemoteAddr, ":") + port, err := strconv.Atoi(remote[1]) + if err != nil { + log.Printf("Could not convert port to integer: %s\n", remote[1]) + w.WriteHeader(503) + return + } + + if err != nil { + fmt.Printf("ERROR: Could not load configuration file: %s\n", err) + return + } + + if config.InsecureRuntime == false { + if port >= 1024 { + log.Panicf("DENIED: Connection coming from non-privledged port: %s\n", req.RemoteAddr) + w.WriteHeader(401) + return + } + } + + node, err := assets.FindByIpaddr(remote[0]) + if err != nil { + fmt.Printf("Could not find node by IP address: %s\n", remote[0]) + w.WriteHeader(404) + return + } + + if node.Fqdn == "" { + log.Printf("UNKNOWN: %15s: %s\n", remote[0], req.URL.Path) + w.WriteHeader(404) + return + } else { + log.Printf("REQ: %15s: %s\n", node.Fqdn, req.URL.Path) + } + + if node.RuntimeOverlay != "" { + fileName := config.RuntimeOverlayImage(node.Fqdn) + + err := sendFile(w, fileName, node.Fqdn) + if err != nil { + log.Printf("ERROR: %s\n", err) + } else { + log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) + } + } else { + w.WriteHeader(503) + log.Printf("ERROR: No 'runtime system-overlay' set for node %s\n", node.Fqdn) + } + + return +} diff --git a/internal/app/warewulfd/warewulfd-reponses/system.go b/internal/app/warewulfd/warewulfd-reponses/system.go new file mode 100644 index 00000000..ffae0cf9 --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/system.go @@ -0,0 +1,34 @@ +package warewulfd_responses + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "log" + "net/http" +) + +func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { + config := config.New() + + node, err := getSanity(req) + if err != nil { + w.WriteHeader(404) + log.Println(err) + return + } + + if node.SystemOverlay != "" { + fileName := config.SystemOverlayImage(node.Fqdn) + + err := sendFile(w, fileName, node.Fqdn) + if err != nil { + log.Printf("ERROR: %s\n", err) + } else { + log.Printf("SEND: %15s: %s\n", node.Fqdn, fileName) + } + } else { + w.WriteHeader(503) + log.Printf("ERROR: No 'system system-overlay' set for node %s\n", node.Fqdn) + } + + return +} diff --git a/internal/app/warewulfd/warewulfd-reponses/util.go b/internal/app/warewulfd/warewulfd-reponses/util.go new file mode 100644 index 00000000..293a6f58 --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/util.go @@ -0,0 +1,55 @@ +package warewulfd_responses + +import ( + "github.com/hpcng/warewulf/internal/pkg/assets" + "github.com/hpcng/warewulf/internal/pkg/errors" + "io" + "log" + "net/http" + "os" + "strconv" + "strings" +) + +func getSanity(req *http.Request) (assets.NodeInfo, error) { + url := strings.Split(req.URL.Path, "/") + + hwaddr := strings.ReplaceAll(url[2], "-", ":") + node, err := assets.FindByHwaddr(hwaddr) + if err != nil { + return node, errors.New("Could not find node by HW address") + } + + if node.Fqdn == "" { + log.Printf("UNKNOWN: %15s: %s\n", hwaddr, req.URL.Path) + return node, errors.New("Unknown node HW address: " + hwaddr) + } else { + log.Printf("REQ: %15s: %s\n", node.Fqdn, req.URL.Path) + } + + return node, nil +} + +func sendFile(w http.ResponseWriter, filename string, sendto string) error { + + fd, err := os.Open(filename) + if err != nil { + return err + } + + FileHeader := make([]byte, 512) + fd.Read(FileHeader) + FileContentType := http.DetectContentType(FileHeader) + FileStat, _ := fd.Stat() + FileSize := strconv.FormatInt(FileStat.Size(), 10) + + w.Header().Set("Content-Disposition", "attachment; filename=kernel") + w.Header().Set("Content-Type", FileContentType) + w.Header().Set("Content-Length", FileSize) + + fd.Seek(0, 0) + io.Copy(w, fd) + + fd.Close() + return nil +} diff --git a/internal/app/warewulfd/warewulfd-reponses/vnfs.go b/internal/app/warewulfd/warewulfd-reponses/vnfs.go new file mode 100644 index 00000000..1967b7ab --- /dev/null +++ b/internal/app/warewulfd/warewulfd-reponses/vnfs.go @@ -0,0 +1,35 @@ +package warewulfd_responses + +import ( + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/vnfs" + "log" + "net/http" +) + +func VnfsSend(w http.ResponseWriter, req *http.Request) { + config := config.New() + + node, err := getSanity(req) + if err != nil { + w.WriteHeader(404) + log.Panicln(err) + return + } + + if node.Vnfs != "" { + v := vnfs.New(node.Vnfs) + + err := sendFile(w, config.VnfsImage(v.NameClean()), node.Fqdn) + if err != nil { + log.Printf("ERROR: %s\n", err) + } else { + log.Printf("SEND: %15s: %s\n", node.Fqdn, config.VnfsImage(v.NameClean())) + } + } else { + w.WriteHeader(503) + log.Printf("ERROR: No VNFS set for node %s\n", node.Fqdn) + } + + return +} From 9c1b28932ec2e92759157c0e123e74b6cf46aa5b Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 23:04:57 -0800 Subject: [PATCH 14/15] Last bit of minor bugfixes and cleanups for a working demo --- Makefile | 2 +- internal/app/wwctl/build/vnfs/containerdir.go | 9 +++++++++ internal/app/wwctl/build/vnfs/docker.go | 5 ++++- internal/pkg/assets/assets.go | 19 ++++++------------- internal/pkg/config/config.go | 10 +++++----- overlays/runtime/default/etc/group.ww | 2 +- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 05c0cf58..81fcb1d8 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ lint: @echo Running golangci-lint... @$(GOLANGCI_LINT) run --build-tags "$(WW_BUILD_GO_BUILD_TAGS)" ./... -all: warewulfd wwctl wwclient +all: vendor warewulfd wwctl wwclient files: all install -d -m 0755 /var/warewulf/ diff --git a/internal/app/wwctl/build/vnfs/containerdir.go b/internal/app/wwctl/build/vnfs/containerdir.go index c3725219..92d433a6 100644 --- a/internal/app/wwctl/build/vnfs/containerdir.go +++ b/internal/app/wwctl/build/vnfs/containerdir.go @@ -33,6 +33,13 @@ func BuildContainerdir(v vnfs.VnfsObject) { return } + wwlog.Printf(wwlog.DEBUG, "Making the directory: %s\n", path.Dir(config.VnfsChroot(v.NameClean()))) + err = os.MkdirAll(path.Dir(config.VnfsChroot(v.NameClean())), 0755) + if err != nil { + fmt.Printf("ERROR: %s\n", err) + return + } + wwlog.Printf(wwlog.DEBUG, "Building VNFS image: '%s' -> '%s'\n", v.Source(), config.VnfsImage(v.NameClean())) err = buildVnfs(v.Source(), config.VnfsImage(v.NameClean())) if err != nil { @@ -44,10 +51,12 @@ func BuildContainerdir(v vnfs.VnfsObject) { _ = os.Remove(config.VnfsChroot(v.NameClean()) + "-link") err = os.Symlink(v.Source(), config.VnfsChroot(v.NameClean())+"-link") if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not create symlink for Chroot: %s\n", err) os.Exit(1) } err = os.Rename(config.VnfsChroot(v.NameClean())+"-link", config.VnfsChroot(v.NameClean())) if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not rename link: %s\n", err) os.Exit(1) } diff --git a/internal/app/wwctl/build/vnfs/docker.go b/internal/app/wwctl/build/vnfs/docker.go index 061e4989..84ba9150 100644 --- a/internal/app/wwctl/build/vnfs/docker.go +++ b/internal/app/wwctl/build/vnfs/docker.go @@ -72,11 +72,12 @@ func BuildDocker(v vnfs.VnfsObject) { _ = os.Remove(config.VnfsImage(v.NameClean()) + "-link") err = os.Symlink(hashDestination, config.VnfsImage(v.NameClean())+"-link") if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) + wwlog.Printf(wwlog.ERROR, "Could not create symlink for Image: %s\n", err) os.Exit(1) } err = os.Rename(config.VnfsImage(v.NameClean())+"-link", config.VnfsImage(v.NameClean())) if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not rename link: %s\n", err) os.Exit(1) } @@ -84,10 +85,12 @@ func BuildDocker(v vnfs.VnfsObject) { _ = os.Remove(config.VnfsChroot(v.NameClean()) + "-link") err = os.Symlink(sourcePath, config.VnfsChroot(v.NameClean())+"-link") if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not create symlink for Chroot: %s\n", err) os.Exit(1) } err = os.Rename(config.VnfsChroot(v.NameClean())+"-link", config.VnfsChroot(v.NameClean())) if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not rename link: %s\n", err) os.Exit(1) } diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 025383e5..66ece834 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -2,6 +2,8 @@ package assets import ( "fmt" + "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/vnfs" "gopkg.in/yaml.v2" "io/ioutil" // "os" @@ -72,6 +74,8 @@ type NodeInfo struct { func FindAllNodes() ([]NodeInfo, error) { var c nodeYaml var ret []NodeInfo + config := config.New() + data, err := ioutil.ReadFile(ConfigFile) if err != nil { @@ -134,19 +138,8 @@ func FindAllNodes() ([]NodeInfo, error) { n.Fqdn = node.Hostname } - /* - if b, _ := regexp.MatchString(`^[a-z\-]+://`, n.Vnfs); b == true { - //if strings.HasPrefix(n.Vnfs, "docker://") { - //TODO: This is a kludge and shouldn't be done here. We need to go back - // and do this from a "vnfs" interface or package - v := vnfs.New(n.Vnfs) - n.VnfsDir = v.Root() - } else if strings.HasPrefix(n.Vnfs, "/"){ - n.VnfsDir = n.Vnfs - } else { - wwlog.Printf(wwlog.ERROR, "Configuration 'Vnfs' is invalid for node: %s\n", n.Fqdn) - } - */ + v := vnfs.New(n.Vnfs) + n.VnfsDir = config.VnfsChroot(v.NameClean()) ret = append(ret, n) } diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 6b759e05..4fadcfe6 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -69,23 +69,23 @@ func (self *Config) RuntimeOverlaySource(overlayName string) (string) { } func (self *Config) KernelImage(kernelVersion string) (string) { - return fmt.Sprintf("%s/kernel/vmlinuz-%s", self.LocalStateDir, kernelVersion) + return fmt.Sprintf("%s/provision/kernel/vmlinuz-%s", self.LocalStateDir, kernelVersion) } func (self *Config) KmodsImage(kernelVersion string) (string) { - return fmt.Sprintf("%s/kernel/kmods-%s.img", self.LocalStateDir, kernelVersion) + return fmt.Sprintf("%s/provision/kernel/kmods-%s.img", self.LocalStateDir, kernelVersion) } func (self *Config) VnfsImage(vnfsNameClean string) (string) { - return fmt.Sprintf("%s/vnfs/%s.img.gz", self.LocalStateDir, vnfsNameClean) + return fmt.Sprintf("%s/provision/vnfs/%s.img.gz", self.LocalStateDir, vnfsNameClean) } func (self *Config) SystemOverlayImage(nodeName string) (string) { - return fmt.Sprintf("%s/overlay/system/%s.img", self.LocalStateDir, nodeName) + return fmt.Sprintf("%s/provision/overlay/system/%s.img", self.LocalStateDir, nodeName) } func (self *Config) RuntimeOverlayImage(nodeName string) (string) { - return fmt.Sprintf("%s/overlay/runtime/%s.img", self.LocalStateDir, nodeName) + return fmt.Sprintf("%s/provision/overlay/runtime/%s.img", self.LocalStateDir, nodeName) } func (self *Config) VnfsChroot(vnfsNameClean string) (string) { diff --git a/overlays/runtime/default/etc/group.ww b/overlays/runtime/default/etc/group.ww index a0ea3854..debf247a 100644 --- a/overlays/runtime/default/etc/group.ww +++ b/overlays/runtime/default/etc/group.ww @@ -1,2 +1,2 @@ {{Include (printf "%s%s" .VnfsDir "/etc/group")}} -{{Include "/etc/passwd"}} +{{Include "/etc/group"}} From 663d3bb653ef07a9da595c4c9d9195e7e42ba184 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 11 Nov 2020 23:32:27 -0800 Subject: [PATCH 15/15] Templated iPxe config properly and added "Kernel Args" node config --- etc/ipxe/default.ipxe | 14 ++--- etc/nodes.conf | 2 + .../app/warewulfd/warewulfd-reponses/ipxe.go | 52 ++++++++++++------- internal/pkg/assets/assets.go | 7 +++ 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/etc/ipxe/default.ipxe b/etc/ipxe/default.ipxe index 8f69ab18..e2aac7a9 100644 --- a/etc/ipxe/default.ipxe +++ b/etc/ipxe/default.ipxe @@ -2,14 +2,14 @@ echo echo ================================================================================ -echo Warewulf v4 now booting: @FQDN@ +echo Warewulf v4 now booting: {{.Fqdn}} echo echo -set base http://@IPADDR@:@PORT@/ +set base http://{{.Ipaddr}}:{{.Port}}/ -kernel ${base}/kernel/@HWADDR@ crashkernel=no quiet || reboot -initrd ${base}/vnfs/@HWADDR@ || reboot -initrd ${base}/kmods/@HWADDR@ || reboot -initrd ${base}/overlay-system/@HWADDR@ || reboot -boot || reboot +kernel ${base}/kernel/{{.Hwaddr}} {{.Kernelargs}} || reboot +initrd ${base}/vnfs/{{.Hwaddr}} || reboot +initrd ${base}/kmods/{{.Hwaddr}} || reboot +initrd ${base}/overlay-system/{{.Hwaddr}} || reboot +boot || reboot diff --git a/etc/nodes.conf b/etc/nodes.conf index 8ca76f27..cba94e39 100644 --- a/etc/nodes.conf +++ b/etc/nodes.conf @@ -9,6 +9,7 @@ nodegroups: n0000: hostname: localtest kernel version: 3.10.0-1127.el7.x86_64 + kernel args: crashkernel=no quiet system overlay: test netdevs: eth0: @@ -23,6 +24,7 @@ nodegroups: runtime overlay: default domain suffix: group2 kernel version: 3.10.0-1127.el7.x86_64 + kernel args: crashkernel=no quiet nodes: n0000: hostname: n0000 diff --git a/internal/app/warewulfd/warewulfd-reponses/ipxe.go b/internal/app/warewulfd/warewulfd-reponses/ipxe.go index 0cb2b262..6a4b44a5 100644 --- a/internal/app/warewulfd/warewulfd-reponses/ipxe.go +++ b/internal/app/warewulfd/warewulfd-reponses/ipxe.go @@ -1,17 +1,28 @@ package warewulfd_responses import ( - "bufio" "fmt" "github.com/hpcng/warewulf/internal/pkg/assets" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/wwlog" "log" "net/http" - "os" "strconv" "strings" + "text/template" ) +type iPxeTemplate struct { + Hostname string + Fqdn string + Vnfs string + Hwaddr string + Ipaddr string + Port string + Kernelargs string +} + + func IpxeSend(w http.ResponseWriter, req *http.Request) { url := strings.Split(req.URL.Path, "/") @@ -29,34 +40,35 @@ func IpxeSend(w http.ResponseWriter, req *http.Request) { } if node.HostName != "" { - log.Printf("IPXE: %15s: %s\n", node.Fqdn, req.URL.Path) - conf := config.New() + log.Printf("IPXE: %15s: %s\n", node.Fqdn, req.URL.Path) + + // TODO: Fix template path to use config package ipxeTemplate := fmt.Sprintf("/etc/warewulf/ipxe/%s.ipxe", node.Ipxe) - sourceFD, err := os.Open(ipxeTemplate) + + tmpl, err := template.ParseFiles(ipxeTemplate) if err != nil { - log.Printf("ERROR: Could not open iPXE Template: %s\n", err) - w.WriteHeader(404) + wwlog.Printf(wwlog.ERROR, "%s\n", err) return } - scanner := bufio.NewScanner(sourceFD) + var replace iPxeTemplate - for scanner.Scan() { - newLine := scanner.Text() + replace.Fqdn = node.Fqdn + replace.Ipaddr = conf.Ipaddr + replace.Port = strconv.Itoa(conf.Port) + replace.Hostname = node.HostName + replace.Hwaddr = url[2] + replace.Vnfs = node.Vnfs + replace.Kernelargs = node.KernelArgs - // TODO: Update this to use templates instead of replaces - newLine = strings.ReplaceAll(newLine, "@HWADDR@", url[2]) - newLine = strings.ReplaceAll(newLine, "@IPADDR@", conf.Ipaddr) - newLine = strings.ReplaceAll(newLine, "@HOSTNAME@", node.HostName) - newLine = strings.ReplaceAll(newLine, "@FQDN@", node.Fqdn) - newLine = strings.ReplaceAll(newLine, "@PORT@", strconv.Itoa(conf.Port)) - // TODO: Add KernelArgs to nodes.conf - //newLine = strings.ReplaceAll(newLine, "@KERNELARGS@", node.KernelArgs) - - fmt.Fprintln(w, newLine) + err = tmpl.Execute(w, replace) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return } + log.Printf("SEND: %15s: %s\n", node.Fqdn, ipxeTemplate) } else { diff --git a/internal/pkg/assets/assets.go b/internal/pkg/assets/assets.go index 66ece834..7aad5b42 100644 --- a/internal/pkg/assets/assets.go +++ b/internal/pkg/assets/assets.go @@ -34,6 +34,7 @@ type nodeGroup struct { RuntimeOverlay string `yaml:"runtime system-overlay""` DomainSuffix string `yaml:"domain suffix"` KernelVersion string `yaml:"kernel version"` + KernelArgs string `yaml:"kernel args"` Nodes map[string]nodeEntry } @@ -45,6 +46,7 @@ type nodeEntry struct { RuntimeOverlay string `yaml:"runtime system-overlay"` DomainSuffix string `yaml:"domain suffix"` KernelVersion string `yaml:"kernel version"` + KernelArgs string `yaml:"kernel args"` IpmiIpaddr string `yaml:"ipmi ipaddr"` NetDevs map[string]netDevs } @@ -68,6 +70,7 @@ type NodeInfo struct { SystemOverlay string RuntimeOverlay string KernelVersion string + KernelArgs string NetDevs map[string]netDevs } @@ -99,6 +102,7 @@ func FindAllNodes() ([]NodeInfo, error) { n.SystemOverlay = group.SystemOverlay n.RuntimeOverlay = group.RuntimeOverlay n.KernelVersion = group.KernelVersion + n.KernelArgs = group.KernelArgs n.DomainName = group.DomainSuffix n.Ipxe = group.Ipxe n.NetDevs = node.NetDevs @@ -121,6 +125,9 @@ func FindAllNodes() ([]NodeInfo, error) { if node.Ipxe != "" { n.Ipxe = node.Ipxe } + if node.KernelArgs != "" { + n.KernelArgs = node.KernelArgs + } if n.RuntimeOverlay == "" { n.RuntimeOverlay = "default"