From dba9de2c8b13499dd533cabdc7942a0b2baa1c8f Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 26 Nov 2020 19:16:55 -0800 Subject: [PATCH] Updates to overlay for NodeDB updates --- internal/pkg/overlay/funcmap.go | 7 +- internal/pkg/overlay/overlay.go | 365 ++++++++++++++++++-------------- 2 files changed, 205 insertions(+), 167 deletions(-) diff --git a/internal/pkg/overlay/funcmap.go b/internal/pkg/overlay/funcmap.go index 0ed1c5cd..572fcef3 100644 --- a/internal/pkg/overlay/funcmap.go +++ b/internal/pkg/overlay/funcmap.go @@ -21,15 +21,16 @@ func templateFileInclude(path string) string { func templateVnfsFileInclude(vnfsname string, filepath string) string { wwlog.Printf(wwlog.DEBUG, "Including VNFS file into template: %s: %s\n", vnfsname, filepath) - v, err := vnfs.Load(vnfsname) - if err != nil { + if vnfsname == "" { + wwlog.Printf(wwlog.WARN, "VNFS not set for template: %s\n", filepath) return "" } + v, _ := vnfs.Load(vnfsname) vnfsDir := v.Chroot if util.IsDir(vnfsDir) == false { - wwlog.Printf(wwlog.WARN, "Template requesting file from non-imported VNFS: %s (%s)\n", vnfsname, filepath) + wwlog.Printf(wwlog.WARN, "Template requires VNFS (%s) to be imported: %s\n", vnfsname, filepath) return "" } wwlog.Printf(wwlog.DEBUG, "IncludeVnfs file from: %s/%s\n", vnfsDir, filepath) diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index e7dee1a7..82e6ab73 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -1,209 +1,246 @@ package overlay import ( - "bytes" "fmt" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/errors" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/util" - "github.com/hpcng/warewulf/internal/pkg/vnfs" "github.com/hpcng/warewulf/internal/pkg/wwlog" - "io" "io/ioutil" "os" "os/exec" "path" + "path/filepath" + "regexp" + "strings" "text/template" ) +type TemplateStruct struct { + Fqdn string + Hostname string + Groupname string + Vnfs string + Netdev map[string]*node.NetDevs +} -func OverlayBuild(nodeList []node.NodeInfo, overlayType string) error { +func BuildSystemOverlay(nodeList []node.NodeInfo) error { + return buildOverlay(nodeList, "system") +} + +func BuildRuntimeOverlay(nodeList []node.NodeInfo) error { + return buildOverlay(nodeList, "runtime") +} + +func FindSystemOverlays() ([]string, error) { + return findAllOverlays("system") +} + +func FindRuntimeOverlays() ([]string, error) { + return findAllOverlays("system") +} + +func SystemOverlayInit(name string) error { + return overlayInit(name, "system") +} + +func RuntimeOverlayInit(name string) error { + return overlayInit(name, "runtime") +} + + + + +func findAllOverlays(overlayType string) ([]string, error) { + config := config.New() + var ret []string + var files []os.FileInfo + var err error + + if overlayType == "system" { + wwlog.Printf(wwlog.DEBUG, "Looking for system overlays...") + files, err = ioutil.ReadDir(config.SystemOverlayDir()) + } else if overlayType == "runtime" { + wwlog.Printf(wwlog.DEBUG, "Looking for runtime overlays...") + files, err = ioutil.ReadDir(config.RuntimeOverlayDir()) + } else { + wwlog.Printf(wwlog.ERROR, "overlayType requested is not supported: %s\n", overlayType) + os.Exit(1) + } + + if err != nil { + return ret, err + } + + for _, file := range files { + wwlog.Printf(wwlog.DEBUG, "Evaluating overlay source: %s\n", file.Name()) + if file.IsDir() == true { + ret = append(ret, file.Name()) + } + } + + return ret, nil +} + + +func overlayInit(name string, overlayType string) error { + var path string config := config.New() - for _, n := range nodeList { - wwlog.Printf(wwlog.DEBUG, "Processing overlay for node: %s\n", n.Fqdn) + if overlayType == "system" { + wwlog.Printf(wwlog.DEBUG, "Looking for system overlays...") + path = config.SystemOverlaySource(name) + } else if overlayType == "runtime" { + wwlog.Printf(wwlog.DEBUG, "Looking for runtime overlays...") + path = config.RuntimeOverlaySource(name) + } else { + wwlog.Printf(wwlog.ERROR, "overlayType requested is not supported: %s\n", overlayType) + os.Exit(1) + } - if util.IsDir(config.VnfsImage(vnfs.CleanName(n.Vnfs))) == false { - wwlog.Printf(wwlog.WARN, "'%s', VNFS not built: %s\n", n.Fqdn, n.Vnfs) -// continue - } + if util.IsDir(path) == true { + return errors.New("Overlay already exists: "+name) + } - var files []node.OverlayEntry + err := os.MkdirAll(path, 0755) - tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-runtime-overlay-") - if err != nil { - continue - } + return err +} - if overlayType == "system" { - wwlog.Printf(wwlog.VERBOSE, "Generating system overlay for node: %s\n", n.Fqdn) - files = n.SystemOverlay - } else if overlayType == "runtime" { - wwlog.Printf(wwlog.VERBOSE, "Generating runtime overlay for node: %s\n", n.Fqdn) - files = n.RuntimeOverlay + + + + + +func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { + config := config.New() + + for _, node := range nodeList { + var t TemplateStruct + var OverlayDir string + var OverlayFile string + + if overlayType == "runtime" { + OverlayDir = config.RuntimeOverlaySource(node.RuntimeOverlay.Get()) + OverlayFile = config.RuntimeOverlayImage(node.Fqdn.Get()) + } else if overlayType == "system" { + OverlayDir = config.SystemOverlaySource(node.RuntimeOverlay.Get()) + OverlayFile = config.SystemOverlayImage(node.Fqdn.Get()) } else { wwlog.Printf(wwlog.ERROR, "overlayType requested is not supported: %s\n", overlayType) os.Exit(1) } - for _, file := range files { - fullPath := path.Join(tmpDir, file.Path) + wwlog.Printf(wwlog.DEBUG, "Processing overlay for node: %s\n", node.Fqdn.Get()) - mode := file.Mode - if mode == 0 { - mode = 0755 - } - - if file.Dir == true { - wwlog.Printf(wwlog.DEBUG, "Creating overlay directory: %s\n", file.Path) - err := os.MkdirAll(fullPath, os.FileMode(mode)) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not create parent directory in overlay: %s\n", file.Path) - continue - } - } else if file.File == true { - wwlog.Printf(wwlog.DEBUG, "Creating overlay file: %s\n", file.Path) - - err := os.MkdirAll(path.Dir(fullPath), 0755) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not create parent directory for overlay file: %s\n", file.Dir) - continue - } - - write, err := os.OpenFile(path.Join(tmpDir, file.Path), os.O_RDWR|os.O_CREATE, os.FileMode(mode)) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - continue - } - - if file.Source != "" { - tmpl, _ := template.New("string").Parse(file.Source) - var tpl bytes.Buffer - _ = tmpl.Execute(&tpl, n) - string := tpl.String() - - source, err := os.Open(string) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - source.Close() - continue - } - _, err = io.Copy(write, source) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - source.Close() - continue - } - source.Close() - } - - for _, s := range file.Sources { - tmpl, _ := template.New("string").Parse(s) - var tpl bytes.Buffer - _ = tmpl.Execute(&tpl, n) - string := tpl.String() - - source, err := os.Open(string) - if err != nil { - wwlog.Printf(wwlog.WARN, "%s\n", err) - source.Close() - continue - } - _, err = io.Copy(write, source) - if err != nil { - wwlog.Printf(wwlog.WARN, "%s\n", err) - source.Close() - continue - } - source.Close() - } - - write.Close() - } else if file.Link == true { - wwlog.Printf(wwlog.DEBUG, "Creating overlay link: %s\n", file.Path) - - if file.Source != "" { - err := os.MkdirAll(path.Dir(fullPath), 0755) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not create parent directory for overlay file: %s\n", file.Path) - continue - } - err = os.Symlink(file.Source, fullPath) - if err != nil { - wwlog.Printf(wwlog.WARN, "Could not create overlay symlink: %s\n", err) - continue - } - } else { - wwlog.Printf(wwlog.WARN, "Could not create link, no source defined: %s\n", file.Path) - continue - } - } else if file.Template == true { - wwlog.Printf(wwlog.DEBUG, "Creating overlay file from template: %s\n", file.Path) - - err := os.MkdirAll(path.Dir(fullPath), 0755) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not create parent directory for overlay file: %s\n", file.Path) - continue - } - - if file.Source != "" { - tmpl, err := template.New(path.Base(file.Source)).Funcs(template.FuncMap{ - "Include": templateFileInclude, - "IncludeFromVnfs": templateVnfsFileInclude, - }).ParseFiles(file.Source) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - continue - } - - w, err := os.OpenFile(fullPath, os.O_RDWR|os.O_CREATE, os.FileMode(mode)) - if err != nil { - w.Close() - wwlog.Printf(wwlog.ERROR, "%s\n", err) - continue - } - - err = tmpl.Execute(w, n) - if err != nil { - w.Close() - wwlog.Printf(wwlog.ERROR, "%s\n", err) - continue - } - w.Close() - } - - } else { - wwlog.Printf(wwlog.ERROR, "Overlay file type error: %s\n", file.Path) - } + t.Fqdn = node.Fqdn.Get() + t.Hostname = node.HostName.Get() + t.Groupname = node.GroupName.Get() + t.Vnfs = node.Vnfs.Get() + t.Netdev = node.NetDevs + if overlayType == "runtime" && node.RuntimeOverlay.Defined() == false { + wwlog.Printf(wwlog.WARN, "Undefined runtime overlay, skipping node: %s\n", node.Fqdn.Get()) + } + if overlayType == "system" && node.SystemOverlay.Defined() == false { + wwlog.Printf(wwlog.WARN, "Undefined system overlay, skipping node: %s\n", node.Fqdn.Get()) } - var overlayFile string - if overlayType == "system" { - overlayFile = config.SystemOverlayImage(n.Fqdn) - } else if overlayType == "runtime" { - overlayFile = config.RuntimeOverlayImage(n.Fqdn) - } - - err = os.MkdirAll(path.Dir(overlayFile), 0755) - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not create parent directory for overlay file: %s\n", overlayFile) + wwlog.Printf(wwlog.DEBUG, "Checking to see if overlay directory exists: %s\n", OverlayDir) + if util.IsDir(OverlayDir) == false { + wwlog.Printf(wwlog.WARN, "%-35s: Skipped (runtime overlay template not found)\n", node.Fqdn.Get()) continue } + wwlog.Printf(wwlog.DEBUG, "Creating parent directory for OverlayFile: %s\n", path.Dir(OverlayFile)) + err := os.MkdirAll(path.Dir(OverlayFile), 0755) + if err != nil { + return err + } - wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", n.Fqdn) + wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlayDir) + err = os.Chdir(OverlayDir) + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not chdir() to OverlayDir: %s\n", OverlayDir) + continue + } - os.Chmod(tmpDir, 0755) - cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, overlayFile) + wwlog.Printf(wwlog.DEBUG, "Creating temporary directory for overlay files\n") + tmpDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-overlay-") + if err != nil { + return err + } + + wwlog.Printf(wwlog.DEBUG, "Walking the file system: %s\n", OverlayDir) + err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) + + err := os.MkdirAll(path.Join(tmpDir, location), info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if filepath.Ext(location) == ".ww" { + wwlog.Printf(wwlog.DEBUG, "Found template file: %s\n", location) + + destFile := strings.TrimSuffix(location, ".ww") + + tmpl, err := template.New(path.Base(location)).Funcs(template.FuncMap{ + "Include": templateFileInclude, + "IncludeFromVnfs": templateVnfsFileInclude, + }).ParseGlob(path.Join(OverlayDir, destFile+".ww*")) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + w, err := os.OpenFile(path.Join(tmpDir, destFile), os.O_RDWR|os.O_CREATE, info.Mode()) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + defer w.Close() + + err = tmpl.Execute(w, t) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b == true { + wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location) + } else { + wwlog.Printf(wwlog.DEBUG, "Found file: %s\n", location) + + err := util.CopyFile(path.Join(OverlayDir, location), path.Join(tmpDir, location)) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + return err + } + + } + + return nil + }) + + + wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", node.Fqdn.Get()) + + cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc -F \"%s\"", tmpDir, OverlayFile) wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd) err = exec.Command("/bin/sh", "-c", cmd).Run() if err != nil { wwlog.Printf(wwlog.ERROR, "Could not generate runtime image overlay: %s\n", err) - os.RemoveAll(tmpDir) continue } - wwlog.Printf(wwlog.INFO, "%-35s: Done\n", n.Fqdn) + wwlog.Printf(wwlog.INFO, "%-35s: Done\n", node.Fqdn.Get()) wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir) os.RemoveAll(tmpDir)