From c859e880ffae48e0cacbe7432f879e1568a713e1 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 29 Sep 2021 21:29:52 -0700 Subject: [PATCH 01/10] Cleanup output during overlay build --- internal/pkg/overlay/overlay.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 9eb4bee1..9ba7625d 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -126,12 +126,12 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { var OverlayFile string if overlayType == "runtime" { - wwlog.Printf(wwlog.VERBOSE, "Building runtime overlay for: %s\n", n.Id.Get()) + wwlog.Printf(wwlog.DEBUG, "Building runtime overlay for: %s\n", n.Id.Get()) OverlayDir = config.RuntimeOverlaySource(n.RuntimeOverlay.Get()) OverlayFile = config.RuntimeOverlayImage(n.Id.Get()) } else if overlayType == "system" { - wwlog.Printf(wwlog.VERBOSE, "Building system overlay for: %s\n", n.Id.Get()) + wwlog.Printf(wwlog.DEBUG, "Building system overlay for: %s\n", n.Id.Get()) OverlayDir = config.SystemOverlaySource(n.SystemOverlay.Get()) OverlayFile = config.SystemOverlayImage(n.Id.Get()) @@ -256,7 +256,7 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { } defer w.Close() - wwlog.Printf(wwlog.VERBOSE, "Writing overlay template: OVERLAY:/%s\n", destFile) + wwlog.Printf(wwlog.DEBUG, "Writing overlay template: OVERLAY:/%s\n", destFile) err = tmpl.Execute(w, t) if err != nil { wwlog.Printf(wwlog.ERROR, "tmpl.Execute %s\n", err) @@ -291,14 +291,14 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { os.Exit(1) } - wwlog.Printf(wwlog.VERBOSE, "Finished generating overlay directory for: %s\n", n.Id.Get()) + wwlog.Printf(wwlog.DEBUG, "Finished generating overlay directory for: %s\n", n.Id.Get()) compressor, err := exec.LookPath("pigz") if err != nil { - wwlog.Printf(wwlog.VERBOSE, "Could not locate PIGZ, using GZIP\n") + wwlog.Printf(wwlog.DEBUG, "Could not locate PIGZ, using GZIP\n") compressor = "gzip" } else { - wwlog.Printf(wwlog.VERBOSE, "Using PIGZ to compress the overlay: %s\n", compressor) + wwlog.Printf(wwlog.DEBUG, "Using PIGZ to compress the overlay: %s\n", compressor) } cmd := fmt.Sprintf("cd \"%s\"; find . | cpio --quiet -o -H newc | %s -c > \"%s\"", tmpDir, compressor, OverlayFile) @@ -309,7 +309,7 @@ func buildOverlay(nodeList []node.NodeInfo, overlayType string) error { wwlog.Printf(wwlog.ERROR, "Could not generate runtime image overlay: %s\n", err) continue } - wwlog.Printf(wwlog.INFO, "%-35s: Done\n", n.Id.Get()) + wwlog.Printf(wwlog.VERBOSE, "%-35s: Done\n", n.Id.Get()) wwlog.Printf(wwlog.DEBUG, "Removing temporary directory: %s\n", tmpDir) os.RemoveAll(tmpDir) From 49a573b92621dfae2f5c1d6a250a73a59c8c4a99 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 29 Sep 2021 21:30:09 -0700 Subject: [PATCH 02/10] Always build overlays on demand by Warewulfd --- internal/pkg/warewulfd/runtime.go | 23 +++++++++++++++-------- internal/pkg/warewulfd/system.go | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index 3f29a2fa..99afb7fb 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -7,6 +7,8 @@ import ( "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" ) @@ -46,30 +48,35 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { } } - node, err := nodes.FindByIpaddr(remote[0]) + n, err := nodes.FindByIpaddr(remote[0]) if err != nil { daemonLogf("WARNING: Could not find node by IP address: %s\n", remote[0]) w.WriteHeader(404) return } - if !node.Id.Defined() { - daemonLogf("REQ: %15s: %s (unknown/unconfigured node)\n", node.Id.Get(), req.URL.Path) + if !n.Id.Defined() { + daemonLogf("REQ: %15s: %s (unknown/unconfigured node)\n", n.Id.Get(), req.URL.Path) w.WriteHeader(404) return } else { - daemonLogf("REQ: %15s: %s\n", node.Id.Get(), req.URL.Path) + daemonLogf("REQ: %15s: %s\n", n.Id.Get(), req.URL.Path) } - if node.RuntimeOverlay.Defined() { - fileName := config.RuntimeOverlayImage(node.Id.Get()) + if n.RuntimeOverlay.Defined() { + fileName := config.RuntimeOverlayImage(n.Id.Get()) - err := sendFile(w, fileName, node.Id.Get()) + if util.PathIsNewer(fileName, node.ConfigFile) { + daemonLogf("BUILD: %15s: Runtime Overlay\n", n.Id.Get()) + _ = overlay.BuildRuntimeOverlay([]node.NodeInfo{n}) + } + + err := sendFile(w, fileName, n.Id.Get()) if err != nil { daemonLogf("ERROR: %s\n", err) } } else { w.WriteHeader(503) - daemonLogf("WARNING: No 'runtime system-overlay' set for node %s\n", node.Id.Get()) + daemonLogf("WARNING: No 'runtime system-overlay' set for node %s\n", n.Id.Get()) } } diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index 293c88ca..7921cfe1 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -4,25 +4,33 @@ import ( "net/http" "github.com/hpcng/warewulf/internal/pkg/config" + "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/overlay" + "github.com/hpcng/warewulf/internal/pkg/util" ) func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { - node, err := getSanity(req) + n, err := getSanity(req) if err != nil { w.WriteHeader(404) daemonLogf("ERROR: %s\n", err) return } - if node.SystemOverlay.Defined() { - fileName := config.SystemOverlayImage(node.Id.Get()) + if n.SystemOverlay.Defined() { + fileName := config.SystemOverlayImage(n.Id.Get()) - err := sendFile(w, fileName, node.Id.Get()) + if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) { + daemonLogf("BUILD: %15s: System Overlay\n", n.Id.Get()) + _ = overlay.BuildSystemOverlay([]node.NodeInfo{n}) + } + + err := sendFile(w, fileName, n.Id.Get()) if err != nil { daemonLogf("ERROR: %s\n", err) } } else { w.WriteHeader(503) - daemonLogf("WARNING: No 'system system-overlay' set for node %s\n", node.Id.Get()) + daemonLogf("WARNING: No 'system system-overlay' set for node %s\n", n.Id.Get()) } } From 8d56a866e1bb61245a43d1c773324fb05e5eccbd Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 29 Sep 2021 21:51:57 -0700 Subject: [PATCH 03/10] also rebuild on overlay internal file changes --- internal/pkg/warewulfd/runtime.go | 2 +- internal/pkg/warewulfd/system.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index 99afb7fb..c29589ab 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -66,7 +66,7 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { if n.RuntimeOverlay.Defined() { fileName := config.RuntimeOverlayImage(n.Id.Get()) - if util.PathIsNewer(fileName, node.ConfigFile) { + if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.RuntimeOverlaySource(n.RuntimeOverlay.Get())) { daemonLogf("BUILD: %15s: Runtime Overlay\n", n.Id.Get()) _ = overlay.BuildRuntimeOverlay([]node.NodeInfo{n}) } diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index 7921cfe1..84344318 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -20,7 +20,7 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { if n.SystemOverlay.Defined() { fileName := config.SystemOverlayImage(n.Id.Get()) - if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) { + if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.SystemOverlaySource(n.SystemOverlay.Get())) { daemonLogf("BUILD: %15s: System Overlay\n", n.Id.Get()) _ = overlay.BuildSystemOverlay([]node.NodeInfo{n}) } From 499582e211d58124054a437f888b52c672b26922 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 29 Sep 2021 22:06:46 -0700 Subject: [PATCH 04/10] Don't rebuild overlays on overlay commands automatically --- internal/app/wwctl/overlay/chmod/main.go | 34 --------------- internal/app/wwctl/overlay/chmod/root.go | 2 - internal/app/wwctl/overlay/chown/main.go | 34 --------------- internal/app/wwctl/overlay/chown/root.go | 2 - internal/app/wwctl/overlay/create/main.go | 34 --------------- internal/app/wwctl/overlay/create/root.go | 2 - internal/app/wwctl/overlay/delete/main.go | 34 --------------- internal/app/wwctl/overlay/delete/root.go | 6 +-- internal/app/wwctl/overlay/edit/main.go | 52 +---------------------- internal/app/wwctl/overlay/edit/root.go | 8 ++-- internal/app/wwctl/overlay/mkdir/main.go | 37 ---------------- internal/app/wwctl/overlay/mkdir/root.go | 4 +- 12 files changed, 7 insertions(+), 242 deletions(-) diff --git a/internal/app/wwctl/overlay/chmod/main.go b/internal/app/wwctl/overlay/chmod/main.go index 1e95b451..cca88b76 100644 --- a/internal/app/wwctl/overlay/chmod/main.go +++ b/internal/app/wwctl/overlay/chmod/main.go @@ -6,8 +6,6 @@ import ( "strconv" "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/pkg/errors" @@ -55,37 +53,5 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } - } - return nil } diff --git a/internal/app/wwctl/overlay/chmod/root.go b/internal/app/wwctl/overlay/chmod/root.go index 4a7431c3..b9a254d6 100644 --- a/internal/app/wwctl/overlay/chmod/root.go +++ b/internal/app/wwctl/overlay/chmod/root.go @@ -11,11 +11,9 @@ var ( RunE: CobraRunE, Args: cobra.ExactArgs(4), } - NoOverlayUpdate bool ) func init() { - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/chown/main.go b/internal/app/wwctl/overlay/chown/main.go index 2e3165a6..5f122e2b 100644 --- a/internal/app/wwctl/overlay/chown/main.go +++ b/internal/app/wwctl/overlay/chown/main.go @@ -6,8 +6,6 @@ import ( "strconv" "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" @@ -69,37 +67,5 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } - } - return nil } diff --git a/internal/app/wwctl/overlay/chown/root.go b/internal/app/wwctl/overlay/chown/root.go index 93218a3a..941328f0 100644 --- a/internal/app/wwctl/overlay/chown/root.go +++ b/internal/app/wwctl/overlay/chown/root.go @@ -11,11 +11,9 @@ var ( RunE: CobraRunE, Args: cobra.RangeArgs(4, 5), } - NoOverlayUpdate bool ) func init() { - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/create/main.go b/internal/app/wwctl/overlay/create/main.go index a6d91356..97570beb 100644 --- a/internal/app/wwctl/overlay/create/main.go +++ b/internal/app/wwctl/overlay/create/main.go @@ -3,7 +3,6 @@ package create import ( "os" - "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/pkg/errors" @@ -32,39 +31,6 @@ func CobraRunE(cmd *cobra.Command, args []string) error { wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } - wwlog.Printf(wwlog.INFO, "Created new runtime overlay: %s\n", overlayName) - } - - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get nodeList: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } } return nil diff --git a/internal/app/wwctl/overlay/create/root.go b/internal/app/wwctl/overlay/create/root.go index 49ff28fa..15242009 100644 --- a/internal/app/wwctl/overlay/create/root.go +++ b/internal/app/wwctl/overlay/create/root.go @@ -12,11 +12,9 @@ var ( RunE: CobraRunE, Args: cobra.ExactArgs(2), } - NoOverlayUpdate bool ) func init() { - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/delete/main.go b/internal/app/wwctl/overlay/delete/main.go index 8463879b..d1b98eb5 100644 --- a/internal/app/wwctl/overlay/delete/main.go +++ b/internal/app/wwctl/overlay/delete/main.go @@ -7,7 +7,6 @@ import ( "github.com/hpcng/warewulf/internal/pkg/config" "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/pkg/errors" @@ -128,39 +127,6 @@ func CobraRunE(cmd *cobra.Command, args []string) error { i = path.Dir(i) } } - fmt.Printf("Deleted from overlay: %s:%s:%s\n", overlayKind, overlayName, overlayPath) - } - - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } } return nil diff --git a/internal/app/wwctl/overlay/delete/root.go b/internal/app/wwctl/overlay/delete/root.go index d38da801..bfe9299c 100644 --- a/internal/app/wwctl/overlay/delete/root.go +++ b/internal/app/wwctl/overlay/delete/root.go @@ -14,15 +14,13 @@ var ( Args: cobra.RangeArgs(2, 3), Aliases: []string{"rm", "del"}, } - Force bool - Parents bool - NoOverlayUpdate bool + Force bool + Parents bool ) func init() { baseCmd.PersistentFlags().BoolVarP(&Force, "force", "f", false, "Force deletion of a non-empty overlay") baseCmd.PersistentFlags().BoolVarP(&Parents, "parents", "p", false, "Remove empty parent directories") - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/edit/main.go b/internal/app/wwctl/overlay/edit/main.go index b6fa531b..d5e1dbb3 100644 --- a/internal/app/wwctl/overlay/edit/main.go +++ b/internal/app/wwctl/overlay/edit/main.go @@ -7,8 +7,6 @@ import ( "path/filepath" "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/pkg/errors" @@ -82,59 +80,11 @@ func CobraRunE(cmd *cobra.Command, args []string) error { fmt.Fprintf(w, "\n") } - shasum1, err := util.ShaSumFile(overlayFile) - if err != nil { - wwlog.Printf(wwlog.WARN, "Could not open overlay file for checksum: %s\n", err) - } - - err = util.ExecInteractive(editor, overlayFile) + err := util.ExecInteractive(editor, overlayFile) if err != nil { wwlog.Printf(wwlog.ERROR, "Editor process existed with non-zero\n") os.Exit(1) } - wwlog.Printf(wwlog.INFO, "Updated: %s:%s:%s\n", overlayKind, overlayName, fileName) - - shasum2, err := util.ShaSumFile(overlayFile) - if err != nil { - wwlog.Printf(wwlog.WARN, "Could not open overlay file for checksum: %s\n", err) - } - - if shasum1 == shasum2 { - wwlog.Printf(wwlog.VERBOSE, "Not updating overlays, no file change\n") - NoOverlayUpdate = true - } - - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } - } return nil } diff --git a/internal/app/wwctl/overlay/edit/root.go b/internal/app/wwctl/overlay/edit/root.go index 64cb7b31..3ec6add2 100644 --- a/internal/app/wwctl/overlay/edit/root.go +++ b/internal/app/wwctl/overlay/edit/root.go @@ -14,17 +14,15 @@ var ( RunE: CobraRunE, Args: cobra.ExactArgs(3), } - ListFiles bool - CreateDirs bool - PermMode int32 - NoOverlayUpdate bool + ListFiles bool + CreateDirs bool + PermMode int32 ) func init() { baseCmd.PersistentFlags().BoolVarP(&ListFiles, "files", "f", false, "List files contained within a given overlay") baseCmd.PersistentFlags().BoolVarP(&CreateDirs, "parents", "p", false, "Create any necessary parent directories") baseCmd.PersistentFlags().Int32VarP(&PermMode, "mode", "m", 0755, "Permission mode for directory") - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/overlay/mkdir/main.go b/internal/app/wwctl/overlay/mkdir/main.go index c072276f..99d30550 100644 --- a/internal/app/wwctl/overlay/mkdir/main.go +++ b/internal/app/wwctl/overlay/mkdir/main.go @@ -1,13 +1,10 @@ package mkdir import ( - "fmt" "os" "path" "github.com/hpcng/warewulf/internal/pkg/config" - "github.com/hpcng/warewulf/internal/pkg/node" - "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/pkg/errors" @@ -46,39 +43,5 @@ func CobraRunE(cmd *cobra.Command, args []string) error { os.Exit(1) } - fmt.Printf("Created directory within overlay: %s:%s:%s\n", overlayKind, overlayName, overlayDir) - - if !NoOverlayUpdate { - n, err := node.New() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not open node configuration: %s\n", err) - os.Exit(1) - } - - nodes, err := n.FindAllNodes() - if err != nil { - wwlog.Printf(wwlog.ERROR, "Could not get node list: %s\n", err) - os.Exit(1) - } - - var updateNodes []node.NodeInfo - - for _, node := range nodes { - if overlayKind == "system" && node.SystemOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } else if overlayKind == "runtime" && node.RuntimeOverlay.Get() == overlayName { - updateNodes = append(updateNodes, node) - } - } - - if overlayKind == "system" { - wwlog.Printf(wwlog.INFO, "Updating System Overlays...\n") - return overlay.BuildSystemOverlay(updateNodes) - } else if overlayKind == "runtime" { - wwlog.Printf(wwlog.INFO, "Updating Runtime Overlays...\n") - return overlay.BuildRuntimeOverlay(updateNodes) - } - } - return nil } diff --git a/internal/app/wwctl/overlay/mkdir/root.go b/internal/app/wwctl/overlay/mkdir/root.go index 88f385b8..8a557079 100644 --- a/internal/app/wwctl/overlay/mkdir/root.go +++ b/internal/app/wwctl/overlay/mkdir/root.go @@ -12,13 +12,11 @@ var ( RunE: CobraRunE, Args: cobra.MinimumNArgs(3), } - PermMode int32 - NoOverlayUpdate bool + PermMode int32 ) func init() { baseCmd.PersistentFlags().Int32VarP(&PermMode, "mode", "m", 0755, "Permission mode for directory") - baseCmd.PersistentFlags().BoolVarP(&NoOverlayUpdate, "noupdate", "n", false, "Don't update overlays") } // GetRootCommand returns the root cobra.Command for the application. From a8f2c58dccd3710d5c7c37e8a8c59ef192cbbe7c Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Wed, 29 Sep 2021 22:48:15 -0700 Subject: [PATCH 05/10] Fix lint error --- internal/app/wwctl/overlay/delete/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/app/wwctl/overlay/delete/main.go b/internal/app/wwctl/overlay/delete/main.go index d1b98eb5..7a3a17a5 100644 --- a/internal/app/wwctl/overlay/delete/main.go +++ b/internal/app/wwctl/overlay/delete/main.go @@ -87,8 +87,6 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "failed to persist node updates") } - overlayName = "default" - } else { removePath := path.Join(overlayPath, fileName) From dffa5a348e6d592c6e22a556f2c4e3c8235c1875 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Sep 2021 23:01:19 -0700 Subject: [PATCH 06/10] Minor bugfix in overlay-runtime REST path --- internal/pkg/warewulfd/warewulfd.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/pkg/warewulfd/warewulfd.go b/internal/pkg/warewulfd/warewulfd.go index 2cf18678..2776590b 100644 --- a/internal/pkg/warewulfd/warewulfd.go +++ b/internal/pkg/warewulfd/warewulfd.go @@ -5,8 +5,8 @@ import ( "net/http" "os" "os/signal" - "syscall" "strconv" + "syscall" "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/pkg/errors" @@ -39,19 +39,19 @@ func RunServer() error { http.HandleFunc("/kmods/", KmodsSend) http.HandleFunc("/container/", ContainerSend) http.HandleFunc("/overlay-system/", SystemOverlaySend) - http.HandleFunc("/overlay-runtime", RuntimeOverlaySend) + http.HandleFunc("/overlay-runtime/", RuntimeOverlaySend) conf, err := warewulfconf.New() if err != nil { - return errors.Wrap(err, "could not get Warewulf configuration") + return errors.Wrap(err, "could not get Warewulf configuration") } - daemonPort := conf.Warewulf.Port + daemonPort := conf.Warewulf.Port daemonLogf("Starting HTTPD REST service on port %d\n", daemonPort) - err = http.ListenAndServe(":" + strconv.Itoa(daemonPort), nil) + err = http.ListenAndServe(":"+strconv.Itoa(daemonPort), nil) if err != nil { - return errors.Wrap(err, "Could not start listening service") + return errors.Wrap(err, "Could not start listening service") } return nil From 2234532ac8e38fdf963f66f3fb105e4ba3e617a9 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Sep 2021 23:01:34 -0700 Subject: [PATCH 07/10] Add dhcpd lease max time --- etc/dhcp/default-dhcpd.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/dhcp/default-dhcpd.conf b/etc/dhcp/default-dhcpd.conf index 6ab76b26..9f92acc2 100644 --- a/etc/dhcp/default-dhcpd.conf +++ b/etc/dhcp/default-dhcpd.conf @@ -35,6 +35,7 @@ if exists user-class and option user-class = "iPXE" { } subnet {{$.Network}} netmask {{$.Netmask}} { + max-lease-time 120; range {{$.RangeStart}} {{$.RangeEnd}}; next-server {{$.Ipaddr}}; } From 384c4c0286452ff57fd02ff40f1020aea913b042 Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Sep 2021 23:02:04 -0700 Subject: [PATCH 08/10] Make the overlay auto-update configurable --- etc/warewulf.conf | 7 ++++--- internal/pkg/warewulfconf/datastructure.go | 7 ++++--- internal/pkg/warewulfconf/modifiers.go | 4 ++-- internal/pkg/warewulfd/runtime.go | 8 +++++--- internal/pkg/warewulfd/system.go | 16 +++++++++++++--- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/etc/warewulf.conf b/etc/warewulf.conf index 8c1aceea..2a75dc3f 100644 --- a/etc/warewulf.conf +++ b/etc/warewulf.conf @@ -1,13 +1,14 @@ -ipaddr: 192.168.1.1 +ipaddr: 192.168.200.1 netmask: 255.255.255.0 warewulf: port: 9873 secure: true + autobuild overlays: true update interval: 60 dhcp: enabled: true - range start: 192.168.1.150 - range end: 192.168.1.200 + range start: 192.168.200.50 + range end: 192.168.200.99 template: default systemd name: dhcpd tftp: diff --git a/internal/pkg/warewulfconf/datastructure.go b/internal/pkg/warewulfconf/datastructure.go index e4464d71..79c5a01c 100644 --- a/internal/pkg/warewulfconf/datastructure.go +++ b/internal/pkg/warewulfconf/datastructure.go @@ -20,9 +20,10 @@ type ControllerConf struct { } type WarewulfConf struct { - Port int `yaml:"port"` - Secure bool `yaml:"secure"` - UpdateInterval int `yaml:"update interval"` + Port int `yaml:"port"` + Secure bool `yaml:"secure"` + UpdateInterval int `yaml:"update interval"` + AutobuildOverlays bool `yaml:"autobuild overlays"` } type DhcpConf struct { diff --git a/internal/pkg/warewulfconf/modifiers.go b/internal/pkg/warewulfconf/modifiers.go index d12eb659..7e87a9ca 100644 --- a/internal/pkg/warewulfconf/modifiers.go +++ b/internal/pkg/warewulfconf/modifiers.go @@ -7,9 +7,9 @@ import ( "gopkg.in/yaml.v2" ) -func (self *ControllerConf) Persist() error { +func (controller *ControllerConf) Persist() error { - out, err := yaml.Marshal(self) + out, err := yaml.Marshal(controller) if err != nil { return err } diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index c29589ab..ea6147cd 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -66,9 +66,11 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { if n.RuntimeOverlay.Defined() { fileName := config.RuntimeOverlayImage(n.Id.Get()) - if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.RuntimeOverlaySource(n.RuntimeOverlay.Get())) { - daemonLogf("BUILD: %15s: Runtime Overlay\n", n.Id.Get()) - _ = overlay.BuildRuntimeOverlay([]node.NodeInfo{n}) + if conf.Warewulf.AutobuildOverlays == true { + if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.RuntimeOverlaySource(n.RuntimeOverlay.Get())) { + daemonLogf("BUILD: %15s: Runtime Overlay\n", n.Id.Get()) + _ = overlay.BuildRuntimeOverlay([]node.NodeInfo{n}) + } } err := sendFile(w, fileName, n.Id.Get()) diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index 84344318..3494dc1d 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -7,9 +7,17 @@ import ( "github.com/hpcng/warewulf/internal/pkg/node" "github.com/hpcng/warewulf/internal/pkg/overlay" "github.com/hpcng/warewulf/internal/pkg/util" + "github.com/hpcng/warewulf/internal/pkg/warewulfconf" ) func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { + conf, err := warewulfconf.New() + if err != nil { + daemonLogf("ERROR: Could not read Warewulf configuration file: %s\n", err) + w.WriteHeader(503) + return + } + n, err := getSanity(req) if err != nil { w.WriteHeader(404) @@ -20,9 +28,11 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { if n.SystemOverlay.Defined() { fileName := config.SystemOverlayImage(n.Id.Get()) - if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.SystemOverlaySource(n.SystemOverlay.Get())) { - daemonLogf("BUILD: %15s: System Overlay\n", n.Id.Get()) - _ = overlay.BuildSystemOverlay([]node.NodeInfo{n}) + if conf.Warewulf.AutobuildOverlays == true { + if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.SystemOverlaySource(n.SystemOverlay.Get())) { + daemonLogf("BUILD: %15s: System Overlay\n", n.Id.Get()) + _ = overlay.BuildSystemOverlay([]node.NodeInfo{n}) + } } err := sendFile(w, fileName, n.Id.Get()) From 238b61a6823d6b73345b706c78b1d7ffd1ccc88b Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Sep 2021 23:40:16 -0700 Subject: [PATCH 09/10] Daemon logging to syslog (#152) --- etc/warewulf.conf | 1 + internal/app/wwctl/server/start/main.go | 7 ++ internal/pkg/warewulfconf/constructors.go | 76 ++++++++++++---------- internal/pkg/warewulfconf/datastructure.go | 1 + internal/pkg/warewulfd/util.go | 31 ++++++++- 5 files changed, 81 insertions(+), 35 deletions(-) diff --git a/etc/warewulf.conf b/etc/warewulf.conf index 2a75dc3f..938c0bc9 100644 --- a/etc/warewulf.conf +++ b/etc/warewulf.conf @@ -5,6 +5,7 @@ warewulf: secure: true autobuild overlays: true update interval: 60 + syslog: false dhcp: enabled: true range start: 192.168.200.50 diff --git a/internal/app/wwctl/server/start/main.go b/internal/app/wwctl/server/start/main.go index 0c184b9a..b1954a46 100644 --- a/internal/app/wwctl/server/start/main.go +++ b/internal/app/wwctl/server/start/main.go @@ -1,13 +1,20 @@ package start import ( + "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/hpcng/warewulf/internal/pkg/warewulfd" "github.com/pkg/errors" "github.com/spf13/cobra" ) func CobraRunE(cmd *cobra.Command, args []string) error { + if SetForeground { + conf, err := warewulfconf.New() + if err != nil { + return errors.Wrap(err, "Could not read Warewulf configuration file") + } + conf.Warewulf.Syslog = false return errors.Wrap(warewulfd.RunServer(), "failed to start Warewulf server") } else { return errors.Wrap(warewulfd.DaemonStart(), "failed to start Warewulf server") diff --git a/internal/pkg/warewulfconf/constructors.go b/internal/pkg/warewulfconf/constructors.go index fc0886f3..76cefbe6 100644 --- a/internal/pkg/warewulfconf/constructors.go +++ b/internal/pkg/warewulfconf/constructors.go @@ -10,43 +10,53 @@ import ( "gopkg.in/yaml.v2" ) +var singleton ControllerConf + func New() (ControllerConf, error) { var ret ControllerConf - wwlog.Printf(wwlog.DEBUG, "Opening Warewulf configuration file: %s\n", ConfigFile) - data, err := ioutil.ReadFile(ConfigFile) - if err != nil { - fmt.Printf("error reading Warewulf configuration file\n") - return ret, err + if (ControllerConf{}) == singleton { + wwlog.Printf(wwlog.DEBUG, "Opening Warewulf configuration file: %s\n", ConfigFile) + data, err := ioutil.ReadFile(ConfigFile) + if err != nil { + fmt.Printf("error reading Warewulf configuration file\n") + return ret, err + } + + wwlog.Printf(wwlog.DEBUG, "Unmarshaling the Warewulf configuration\n") + err = yaml.Unmarshal(data, &ret) + if err != nil { + return ret, err + } + + if ret.Ipaddr == "" { + wwlog.Printf(wwlog.WARN, "IP address is not configured in warewulfd.conf\n") + } + if ret.Netmask == "" { + wwlog.Printf(wwlog.WARN, "Netmask is not configured in warewulfd.conf\n") + } + + if ret.Network == "" { + mask := net.IPMask(net.ParseIP(ret.Netmask).To4()) + size, _ := mask.Size() + + sub := ipsubnet.SubnetCalculator(ret.Ipaddr, size) + + ret.Network = sub.GetNetworkPortion() + } + + if ret.Warewulf.Port == 0 { + ret.Warewulf.Port = 9873 + } + + wwlog.Printf(wwlog.DEBUG, "Returning warewulf config object\n") + singleton = ret + + } else { + wwlog.Printf(wwlog.DEBUG, "Returning cached warewulf config object\n") + + ret = singleton } - wwlog.Printf(wwlog.DEBUG, "Unmarshaling the Warewulf configuration\n") - err = yaml.Unmarshal(data, &ret) - if err != nil { - return ret, err - } - - if ret.Ipaddr == "" { - wwlog.Printf(wwlog.WARN, "IP address is not configured in warewulfd.conf\n") - } - if ret.Netmask == "" { - wwlog.Printf(wwlog.WARN, "Netmask is not configured in warewulfd.conf\n") - } - - if ret.Network == "" { - mask := net.IPMask(net.ParseIP(ret.Netmask).To4()) - size, _ := mask.Size() - - sub := ipsubnet.SubnetCalculator(ret.Ipaddr, size) - - ret.Network = sub.GetNetworkPortion() - } - - if ret.Warewulf.Port == 0 { - ret.Warewulf.Port = 9873 - } - - wwlog.Printf(wwlog.DEBUG, "Returning node object\n") - return ret, nil } diff --git a/internal/pkg/warewulfconf/datastructure.go b/internal/pkg/warewulfconf/datastructure.go index 79c5a01c..8315ddd6 100644 --- a/internal/pkg/warewulfconf/datastructure.go +++ b/internal/pkg/warewulfconf/datastructure.go @@ -24,6 +24,7 @@ type WarewulfConf struct { Secure bool `yaml:"secure"` UpdateInterval int `yaml:"update interval"` AutobuildOverlays bool `yaml:"autobuild overlays"` + Syslog bool `yaml:"syslog"` } type DhcpConf struct { diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index f88514bb..c7c4991b 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -3,6 +3,8 @@ package warewulfd import ( "fmt" "io" + "log" + "log/syslog" "net/http" "os" "strconv" @@ -10,12 +12,37 @@ import ( "time" "github.com/hpcng/warewulf/internal/pkg/node" + "github.com/hpcng/warewulf/internal/pkg/warewulfconf" "github.com/pkg/errors" ) +var logwriter syslog.Writer +var loginit bool + func daemonLogf(message string, a ...interface{}) { - prefix := fmt.Sprintf("[%s] ", time.Now().Format(time.UnixDate)) - fmt.Printf(prefix+message, a...) + conf, err := warewulfconf.New() + if err != nil { + fmt.Printf("ERROR: Could not read Warewulf configuration file: %s\n", err) + return + } + + if conf.Warewulf.Syslog == true { + if loginit == false { + logwriter, err := syslog.New(syslog.LOG_NOTICE, "warewulfd") + if err != nil { + return + } + log.SetOutput(logwriter) + loginit = true + } + log.SetFlags(0) + log.SetPrefix("") + + log.Printf(message, a...) + } else { + prefix := fmt.Sprintf("[%s] ", time.Now().Format(time.UnixDate)) + fmt.Printf(prefix+message, a...) + } } func getSanity(req *http.Request) (node.NodeInfo, error) { From 6e78f1e89b7a26ccd64f0d899f3111bebe83f55a Mon Sep 17 00:00:00 2001 From: Gregory Kurtzer Date: Thu, 30 Sep 2021 23:43:24 -0700 Subject: [PATCH 10/10] Fix lint errors --- internal/pkg/warewulfd/runtime.go | 8 ++++---- internal/pkg/warewulfd/system.go | 2 +- internal/pkg/warewulfd/util.go | 13 ++++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index ea6147cd..eaf17eef 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -22,7 +22,7 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { nodes, err := node.New() if err != nil { - daemonLogf("%s | ERROR: Could not read node configuration file: %s\n", err) + daemonLogf("ERROR: Could not read node configuration file: %s\n", err) w.WriteHeader(503) return } @@ -30,13 +30,13 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { remote := strings.Split(req.RemoteAddr, ":") port, err := strconv.Atoi(remote[1]) if err != nil { - daemonLogf("%s | ERROR: Could not convert port to integer: %s\n", remote[1]) + daemonLogf("ERROR: Could not convert port to integer: %s\n", remote[1]) w.WriteHeader(503) return } if err != nil { - daemonLogf("%s | ERROR: Could not load configuration file: %s\n", err) + daemonLogf("ERROR: Could not load configuration file: %s\n", err) return } @@ -66,7 +66,7 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { if n.RuntimeOverlay.Defined() { fileName := config.RuntimeOverlayImage(n.Id.Get()) - if conf.Warewulf.AutobuildOverlays == true { + if conf.Warewulf.AutobuildOverlays { if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.RuntimeOverlaySource(n.RuntimeOverlay.Get())) { daemonLogf("BUILD: %15s: Runtime Overlay\n", n.Id.Get()) _ = overlay.BuildRuntimeOverlay([]node.NodeInfo{n}) diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index 3494dc1d..f63fdac1 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -28,7 +28,7 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { if n.SystemOverlay.Defined() { fileName := config.SystemOverlayImage(n.Id.Get()) - if conf.Warewulf.AutobuildOverlays == true { + if conf.Warewulf.AutobuildOverlays { if !util.IsFile(fileName) || util.PathIsNewer(fileName, node.ConfigFile) || util.PathIsNewer(fileName, config.SystemOverlaySource(n.SystemOverlay.Get())) { daemonLogf("BUILD: %15s: System Overlay\n", n.Id.Get()) _ = overlay.BuildSystemOverlay([]node.NodeInfo{n}) diff --git a/internal/pkg/warewulfd/util.go b/internal/pkg/warewulfd/util.go index c7c4991b..840b1ef1 100644 --- a/internal/pkg/warewulfd/util.go +++ b/internal/pkg/warewulfd/util.go @@ -16,7 +16,7 @@ import ( "github.com/pkg/errors" ) -var logwriter syslog.Writer +var logwriter *syslog.Writer var loginit bool func daemonLogf(message string, a ...interface{}) { @@ -26,19 +26,22 @@ func daemonLogf(message string, a ...interface{}) { return } - if conf.Warewulf.Syslog == true { - if loginit == false { - logwriter, err := syslog.New(syslog.LOG_NOTICE, "warewulfd") + if conf.Warewulf.Syslog { + if !loginit { + var err error + + logwriter, err = syslog.New(syslog.LOG_NOTICE, "warewulfd") if err != nil { return } log.SetOutput(logwriter) loginit = true } + log.SetFlags(0) log.SetPrefix("") - log.Printf(message, a...) + } else { prefix := fmt.Sprintf("[%s] ", time.Now().Format(time.UnixDate)) fmt.Printf(prefix+message, a...)