From 9dbf3711741e5c40531218cb1753dd909aa012a5 Mon Sep 17 00:00:00 2001 From: Christian Goll Date: Wed, 16 Mar 2022 12:23:33 +0100 Subject: [PATCH] overlay build allow output to given dir --- internal/app/wwctl/overlay/build/main.go | 35 ++++-- internal/app/wwctl/overlay/build/root.go | 6 +- internal/pkg/overlay/overlay.go | 131 +++++++++++++---------- 3 files changed, 102 insertions(+), 70 deletions(-) diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 2c348479..eaeeaac7 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -1,7 +1,9 @@ package build import ( + "errors" "os" + "strings" "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" @@ -11,7 +13,29 @@ import ( ) func CobraRunE(cmd *cobra.Command, args []string) error { + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) + os.Exit(1) + } + nodes, err := nodeDB.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) + os.Exit(1) + } + if OverlayDir != "" { + if OverlayName == "" { + return errors.New("no overlay name given") + } + if len(args) > 0 { + args = hostlist.Expand(args) + for _, node := range nodes { + return overlay.BuildOverlayIndir(node, strings.Split(OverlayName, ","), OverlayDir) + } + } + + } if BuildHost || (!BuildHost && !BuildNodes && len(args) == 0) { err := overlay.BuildHostOverlay() if err != nil { @@ -19,17 +43,6 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } } if BuildNodes || (!BuildHost && !BuildNodes) { - nodeDB, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := nodeDB.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } if len(args) > 0 { args = hostlist.Expand(args) diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index c549a25b..e8ad090e 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -13,12 +13,16 @@ var ( BuildHost bool BuildNodes bool OverlayName string + OverlayDir string ) func init() { baseCmd.PersistentFlags().BoolVarP(&BuildHost, "host", "H", false, "Build overlays only for the host") baseCmd.PersistentFlags().BoolVarP(&BuildNodes, "nodes", "N", false, "Build overlays only for the nodes") - baseCmd.PersistentFlags().StringVarP(&OverlayName, "overlay", "o", "", "Build only specific overlay") + baseCmd.PersistentFlags().StringVarP(&OverlayName, "overlay", "O", "", "Build only specific overlay") + baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay to image, for distribution but write to| + the given directory. An overlay must also be ge given to use this option. '/dev/stdin' will print| + the processed overlay.`) } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 0315d1a8..f6ed7658 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -89,7 +89,7 @@ func BuildHostOverlay() error { wwlog.Printf(wwlog.INFO, "Building overlay for %s: host\n", hostname) idEntry.Set(hostname) host.Id = idEntry - return BuildOverlay(host, []string{"host"}) + return BuildOverlayIndir(host, []string{"host"}, "/") } /* @@ -115,7 +115,7 @@ func FindOverlays() ([]string, error) { } /* -Creates an emtpy overlay +Creates an empty overlay */ func OverlayInit(overlayName string) error { path := OverlaySourceDir(overlayName) @@ -130,48 +130,85 @@ func OverlayInit(overlayName string) error { } /* -Build the given overlays for a node. If the overlayName is = "host", the host overlay is build +Build the given overlays for a node and create a Image for them */ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { + // create the dir where the overlay images will reside + overlayImage := OverlayImage(nodeInfo.Id.Get(), overlayNames) + overlayImageDir := path.Dir(overlayImage) + err := os.MkdirAll(overlayImageDir, 0755) + if err == nil { + wwlog.Printf(wwlog.DEBUG, "Created parent directory for Overlay Images: %s\n", overlayImageDir) + } else { + return errors.Wrap(err, "could not create overlay image directory") + } + + outputDir, err := ioutil.TempDir(os.TempDir(), ".wwctl-overlay-") + if err == nil { + wwlog.Printf(wwlog.DEBUG, "Creating temporary directory for overlay files: %s\n", outputDir) + } else { + return errors.Wrap(err, "could not create overlay temporary directory") + } + err = BuildOverlayIndir(nodeInfo, overlayNames, outputDir) + if err == nil { + return err + } + + wwlog.Printf(wwlog.DEBUG, "Finished generating overlay working directory for: %s/%v\n", nodeInfo.Id.Get(), overlayNames) + compressor, err := exec.LookPath("pigz") + if err != nil { + wwlog.Printf(wwlog.DEBUG, "Could not locate PIGZ, using GZIP\n") + compressor = "gzip" + } else { + wwlog.Printf(wwlog.DEBUG, "Using PIGZ to compress the overlay: %s\n", compressor) + } + + cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc | %s -c > \"%s\"", outputDir, compressor, overlayImage) + + wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd) + err = exec.Command("/bin/sh", "-c", cmd).Run() + if err != nil { + return errors.Wrap(err, "could not generate compressed runtime image overlay") + } + wwlog.Printf(wwlog.VERBOSE, "Completed building overlay image: %s\n", overlayImage) + + wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", outputDir) + os.RemoveAll(outputDir) + return nil +} + +/* +Build the given overlays for a node in the given directory. If the given does not +exists it will be created. +*/ +func BuildOverlayIndir(nodeInfo node.NodeInfo, overlayNames []string, outputDir string) error { if len(overlayNames) == 0 { return errors.New("At least one valid overlay is needed to build for a node\n") } + if !util.IsDir(outputDir) { + return errors.New(fmt.Sprintf("output %s must a be a directory\n", outputDir)) + } controller, err := warewulfconf.New() if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } - nodeDB, _ := node.New() - allNodes, _ := nodeDB.FindAllNodes() - var overlayImage string = "/" - if overlayNames[0] != "host" { - overlayImage = OverlayImage(nodeInfo.Id.Get(), overlayNames) + nodeDB, err := node.New() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) + } + allNodes, err := nodeDB.FindAllNodes() + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + os.Exit(1) } - var tstruct TemplateStruct - OverlayImageDir := path.Dir(overlayImage) if !util.ValidString(strings.Join(overlayNames, ""), "^[a-zA-Z0-9-._:]+$") { return errors.New(fmt.Sprintf("overlay names contains illegal characters: %v", overlayNames)) } - var destDir = "/" - if overlayNames[0] != "host" { - err = os.MkdirAll(OverlayImageDir, 0755) - if err == nil { - wwlog.Printf(wwlog.DEBUG, "Created parent directory for Overlay Images: %s\n", OverlayImageDir) - } else { - return errors.Wrap(err, "could not create overlay image directory") - } - - destDir, err = ioutil.TempDir(os.TempDir(), ".wwctl-overlay-") - if err == nil { - wwlog.Printf(wwlog.DEBUG, "Creating temporary directory for overlay files: %s\n", destDir) - } else { - return errors.Wrap(err, "could not create overlay temporary directory") - } - } - wwlog.Printf(wwlog.VERBOSE, "Processing node/overlay: %s/%s\n", nodeInfo.Id.Get(), strings.Join(overlayNames, "-")) - + var tstruct TemplateStruct tstruct.Id = nodeInfo.Id.Get() tstruct.Hostname = nodeInfo.Id.Get() tstruct.Id = nodeInfo.Id.Get() @@ -257,11 +294,11 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { if info.IsDir() { wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) - err = os.MkdirAll(path.Join(destDir, location), info.Mode()) + err = os.MkdirAll(path.Join(outputDir, location), info.Mode()) if err != nil { return errors.Wrap(err, "could not create directory within overlay") } - err = util.CopyUIDGID(location, path.Join(destDir, location)) + err = util.CopyUIDGID(location, path.Join(outputDir, location)) if err != nil { return errors.Wrap(err, "failed setting permissions on overlay directory") } @@ -306,15 +343,15 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { } if writeFile { if backupFile { - if !util.IsFile(path.Join(destDir, destFile+".wwbackup")) && util.IsFile(path.Join(destDir, destFile)) { - err = util.CopyFile(path.Join(destDir, destFile), path.Join(destDir, destFile+".wwbackup")) + if !util.IsFile(path.Join(outputDir, destFile+".wwbackup")) && util.IsFile(path.Join(outputDir, destFile)) { + err = util.CopyFile(path.Join(outputDir, destFile), path.Join(outputDir, destFile+".wwbackup")) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) } } } - w, err := os.OpenFile(path.Join(destDir, destFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) + w, err := os.OpenFile(path.Join(outputDir, destFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) if err != nil { return errors.Wrap(err, "could not open new file for template") } @@ -326,7 +363,7 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { return errors.Wrap(err, "could not write file from template") } - err = util.CopyUIDGID(location, path.Join(destDir, destFile)) + err = util.CopyUIDGID(location, path.Join(outputDir, destFile)) if err != nil { return errors.Wrap(err, "failed setting permissions on template output file") } @@ -342,13 +379,13 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) } - err = os.Symlink(destination, path.Join(destDir, location)) + err = os.Symlink(destination, path.Join(outputDir, location)) if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) } } else { - err := util.CopyFile(location, path.Join(destDir, location)) + err := util.CopyFile(location, path.Join(outputDir, location)) if err == nil { wwlog.Printf(wwlog.DEBUG, "Copied file into overlay: %s\n", location) } else { @@ -363,27 +400,5 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { } } - wwlog.Printf(wwlog.DEBUG, "Finished generating overlay working directory for: %s/%v\n", nodeInfo.Id.Get(), overlayNames) - if overlayNames[0] != "host" { - compressor, err := exec.LookPath("pigz") - if err != nil { - wwlog.Printf(wwlog.DEBUG, "Could not locate PIGZ, using GZIP\n") - compressor = "gzip" - } else { - wwlog.Printf(wwlog.DEBUG, "Using PIGZ to compress the overlay: %s\n", compressor) - } - - cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc | %s -c > \"%s\"", destDir, compressor, overlayImage) - - wwlog.Printf(wwlog.DEBUG, "RUNNING: %s\n", cmd) - err = exec.Command("/bin/sh", "-c", cmd).Run() - if err != nil { - return errors.Wrap(err, "could not generate compressed runtime image overlay") - } - wwlog.Printf(wwlog.VERBOSE, "Completed building overlay image: %s\n", overlayImage) - - wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", destDir) - os.RemoveAll(destDir) - } return nil }