diff --git a/CHANGELOG.md b/CHANGELOG.md index 9616988d..dd060656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,32 +45,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- Document reclaiming RAM by moving the image to swap - New `--partwipe` flag for profile and node set - Updated arguments for `ValidString` to match `regexp.MatchString` - New `mig` overlay to configure NVIDIA MIG devices. #2102 - Documented that booting a node twice fixes broken partition tables -- Updated arguments for `ValidString` to match `regexp.MatchString` -- New `mig` overlay to configure NVIDIA MIG devices. #2102 -- TLS with the command `wwctl configure tls` for key management. - Keys can be created automtically or imported. The runtime overlay is - if TLS is enabled is not distributed over plain http. - -- Documented that booting a node twice fixes broken partition tables -- TLS support for `warewulfd` and REST API +- TLS support for `warewulfd` and REST API. - New `wwctl configure tls` command to generate and configure TLS keys and - certificates + certificates. - New dedicated `warewulfd` server routes (`/ipxe/`, `/kernel/`, `/image/`, - `/initramfs/`, `/system/`, `/runtime/`, `/grub/`, `/efiboot/`) + `/initramfs/`, `/system/`, `/runtime/`, `/grub/`, `/efiboot/`). +- Documented how to detect shadowed overlays in troubleshooting. +- Documented `wwctl clean`. #2034 +- Documented the status of various top-level Warewulf features. +- Initial one-page quick reference guide +- Document configuring the arp cache for large clusters +- Expand troubleshooting documentation for container runtimes ### Changed - Runtime overlay download failure during dracut/wwinit boot is now non-fatal; the node continues to boot and `wwclient` retries the download at runtime. - `hosts` overlay added to the default system overlay list - -### Changed - +- Distinguish between OS images and overlay images in documentation - Updated `MAINTAINING.md` to document golang version policy +- Clarified functionality of syncuser commands and overlay in documentation +- Audit and correct documentation, cobra help text, and log messages for accuracy ## v4.6.5, 2026-01-12 diff --git a/containers/README.md b/containers/README.md index 5782dc7d..46afbb40 100644 --- a/containers/README.md +++ b/containers/README.md @@ -1,6 +1,6 @@ -# Node images have moved +# OS images have moved -Warewulf node images are now being maintained in a separate repository: +Warewulf OS images are now being maintained in a separate repository: https://github.com/warewulf/warewulf-node-images. diff --git a/internal/app/wwctl/configure/hostfile/root.go b/internal/app/wwctl/configure/hostfile/root.go index 8160d19b..d222f022 100644 --- a/internal/app/wwctl/configure/hostfile/root.go +++ b/internal/app/wwctl/configure/hostfile/root.go @@ -8,7 +8,7 @@ import ( var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "hostfile [OPTIONS]", + Use: "hostfile [OPTIONS]", Short: "update hostfile on master", Long: "Manage the hostfile on the master node\n", RunE: CobraRunE, diff --git a/internal/app/wwctl/genconf/warewulfconf/print/root.go b/internal/app/wwctl/genconf/warewulfconf/print/root.go index a1f7ce0c..1362fca0 100644 --- a/internal/app/wwctl/genconf/warewulfconf/print/root.go +++ b/internal/app/wwctl/genconf/warewulfconf/print/root.go @@ -8,7 +8,7 @@ import ( var ( baseCmd = &cobra.Command{ Use: "print", - Short: "print wareweulf.conf", + Short: "print warewulf.conf", Long: "This command prints the actual used warewulf.conf, can be used to create an empty valid warewulf.conf", RunE: CobraRunE, Args: cobra.NoArgs, diff --git a/internal/app/wwctl/image/imprt/root.go b/internal/app/wwctl/image/imprt/root.go index 5f882507..b452fcb7 100644 --- a/internal/app/wwctl/image/imprt/root.go +++ b/internal/app/wwctl/image/imprt/root.go @@ -44,13 +44,13 @@ func init() { baseCmd.PersistentFlags().BoolVarP(&SetUpdate, "update", "u", false, "Overwrite files in an existing image with the files of remote image") baseCmd.PersistentFlags().BoolVarP(&SetBuild, "build", "b", false, "Build image after pulling") baseCmd.PersistentFlags().BoolVar(&SyncUser, "syncuser", false, "Synchronize UIDs/GIDs from host to image") - baseCmd.PersistentFlags().BoolVar(&OciNoHttps, "nohttps", false, "Ignore wrong TLS certificates, superseedes env WAREWULF_OCI_NOHTTPS") - baseCmd.PersistentFlags().StringVar(&OciUsername, "username", "", "Set username for the access to the registry, superseedes env WAREWULF_OCI_USERNAME") - baseCmd.PersistentFlags().StringVar(&OciPassword, "password", "", "Set password for the access to the registry, superseedes env WAREWULF_OCI_PASSWORD") - baseCmd.PersistentFlags().StringVar(&OciPassword, "passwd", "", "Set password for the access to the registry, superseedes env WAREWULF_OCI_PASSWORD") + baseCmd.PersistentFlags().BoolVar(&OciNoHttps, "nohttps", false, "Ignore wrong TLS certificates, supersedes env WAREWULF_OCI_NOHTTPS") + baseCmd.PersistentFlags().StringVar(&OciUsername, "username", "", "Set username for the access to the registry, supersedes env WAREWULF_OCI_USERNAME") + baseCmd.PersistentFlags().StringVar(&OciPassword, "password", "", "Set password for the access to the registry, supersedes env WAREWULF_OCI_PASSWORD") + baseCmd.PersistentFlags().StringVar(&OciPassword, "passwd", "", "Set password for the access to the registry, supersedes env WAREWULF_OCI_PASSWORD") _ = baseCmd.PersistentFlags().MarkHidden("passwd") - baseCmd.PersistentFlags().StringVar(&Platform, "platform", "", "Set other hardware platform e.g. amd64 or arm64, superseedes env WAREWULF_OCI_PLATFORM") - baseCmd.PersistentFlags().StringVar(&Platform, "arch", "", "Set other hardware platform, superseedes env WAREWULF_OCI_PLATFORM") + baseCmd.PersistentFlags().StringVar(&Platform, "platform", "", "Set other hardware platform e.g. amd64 or arm64, supersedes env WAREWULF_OCI_PLATFORM") + baseCmd.PersistentFlags().StringVar(&Platform, "arch", "", "Set other hardware platform, supersedes env WAREWULF_OCI_PLATFORM") _ = baseCmd.PersistentFlags().MarkHidden("arch") } diff --git a/internal/app/wwctl/image/root.go b/internal/app/wwctl/image/root.go index f26fecf9..ffdf2f09 100644 --- a/internal/app/wwctl/image/root.go +++ b/internal/app/wwctl/image/root.go @@ -19,11 +19,9 @@ var baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, Use: "image COMMAND [OPTIONS]", Short: "Operating system image management", - Long: "Starting with version 4, Warewulf uses images to build the bootable\n" + - "node images. These commands will help you import, manage, and transform\n" + - "images into bootable Warewulf images.", - Aliases: []string{"vnfs", "container"}, - Args: cobra.NoArgs, + Long: "Import, manage, and transform OS images as bootable Warewulf images.", + Aliases: []string{"vnfs", "container"}, + Args: cobra.NoArgs, } func init() { diff --git a/internal/app/wwctl/image/syncuser/main.go b/internal/app/wwctl/image/syncuser/main.go index 8be540b6..7ac3c0d3 100644 --- a/internal/app/wwctl/image/syncuser/main.go +++ b/internal/app/wwctl/image/syncuser/main.go @@ -22,7 +22,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { if write && !build { // when write = true and build = false, we will print a warnning, this is the default case - wwlog.Warn("Syncuser is completed. Rebuild image or add `--build` flag for automatic rebuild after syncuser.") + wwlog.Info("Syncuser is completed. Rebuild image or add `--build` flag for automatic rebuild after syncuser.") } else if write && build { // if write = true and build = true, then it'll trigger the image build after sync cbp := &wwapiv1.ImageBuildParameter{ diff --git a/internal/app/wwctl/image/syncuser/root.go b/internal/app/wwctl/image/syncuser/root.go index 1efae9b0..70ab44b0 100644 --- a/internal/app/wwctl/image/syncuser/root.go +++ b/internal/app/wwctl/image/syncuser/root.go @@ -9,10 +9,15 @@ var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, Use: "syncuser [OPTIONS] IMAGE", - Short: "Synchronizes user in image", - Long: `Synchronize the uids and gids from the host to the image. -Users/groups which are only present in the image will be preserved if no -uid/gid collision is detected. File ownerships are also changed.`, + Short: "Synchronize UIDs/GIDs from the server to an OS image", + Long: `Synchronize UIDs and GIDs from the server into an OS image, updating +/etc/passwd, /etc/group, and file ownerships within the image. Users and +groups that exist only in the image are preserved unless a UID/GID collision +is detected. + +This command affects the image itself (a one-time operation at build/import +time). To also make host users available on provisioned nodes at runtime, add +the "syncuser" overlay to the node or profile.`, RunE: CobraRunE, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { @@ -29,8 +34,8 @@ uid/gid collision is detected. File ownerships are also changed.`, ) func init() { - baseCmd.PersistentFlags().BoolVar(&write, "write", true, "Synchronize uis/gids and write files in image") - baseCmd.PersistentFlags().BoolVar(&build, "build", false, "Build image after syncuser is completed") + baseCmd.PersistentFlags().BoolVar(&write, "write", true, "Synchronize UIDs/GIDs and write files in OS image") + baseCmd.PersistentFlags().BoolVar(&build, "build", false, "Build OS image after syncuser is completed") } // GetRootCommand returns the root cobra.Command for the application. diff --git a/internal/app/wwctl/node/export/root.go b/internal/app/wwctl/node/export/root.go index bc62d306..db44d9d1 100644 --- a/internal/app/wwctl/node/export/root.go +++ b/internal/app/wwctl/node/export/root.go @@ -8,7 +8,7 @@ import ( var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "export NODENAME", + Use: "export NODENAME", Short: "Export nodes as yaml to stdout", Long: "This command exports the given nodes as yaml to stdout.", RunE: CobraRunE, diff --git a/internal/app/wwctl/node/root.go b/internal/app/wwctl/node/root.go index c71e1451..a62f9ba5 100644 --- a/internal/app/wwctl/node/root.go +++ b/internal/app/wwctl/node/root.go @@ -17,7 +17,7 @@ import ( var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "node COMMAND [OPTONS]", + Use: "node COMMAND [OPTIONS]", Short: "Node management", Long: "Management of node settings. All node ranges can use brackets to identify\n" + "node ranges. For example: n00[00-4].cluster[0-1] will identify the first 5 nodes\n" + diff --git a/internal/app/wwctl/overlay/build/root.go b/internal/app/wwctl/overlay/build/root.go index 041da655..6cf93723 100644 --- a/internal/app/wwctl/overlay/build/root.go +++ b/internal/app/wwctl/overlay/build/root.go @@ -27,7 +27,7 @@ func init() { panic(err) } baseCmd.PersistentFlags().StringVarP(&OverlayDir, "output", "o", "", `Do not create an overlay image for distribution but write to - the given directory. An overlay must also be ge given to use this option.`) + the given directory. An overlay must also be given to use this option.`) baseCmd.PersistentFlags().IntVar(&Workers, "workers", 0, "The number of parallel workers building overlays (<=0 indicates 1 worker per CPU)") } diff --git a/internal/app/wwctl/profile/edit/root.go b/internal/app/wwctl/profile/edit/root.go index 4b4c323b..0c72ea66 100644 --- a/internal/app/wwctl/profile/edit/root.go +++ b/internal/app/wwctl/profile/edit/root.go @@ -8,8 +8,8 @@ import ( var ( baseCmd = &cobra.Command{ DisableFlagsInUseLine: true, - Use: "edit [OPTIONS] NODENAME", - Short: "Edit node(s) with editor", + Use: "edit [OPTIONS] PROFILE", + Short: "Edit profile(s) with editor", Long: "This command opens an editor for the given profiles.", RunE: CobraRunE, ValidArgsFunction: completions.Profiles, diff --git a/internal/pkg/api/node/set.go b/internal/pkg/api/node/set.go index c19ec97f..62e26209 100644 --- a/internal/pkg/api/node/set.go +++ b/internal/pkg/api/node/set.go @@ -88,7 +88,7 @@ func NodeSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodesYaml if set.PartitionDelete != "" { for diskname, disk := range nodePtr.Disks { if _, ok := disk.Partitions[set.PartitionDelete]; ok { - wwlog.Verbose("Node: %s, on disk %, deleting partition: %s", nId, diskname, set.PartitionDelete) + wwlog.Verbose("Node: %s, on disk %s, deleting partition: %s", nId, diskname, set.PartitionDelete) delete(disk.Partitions, set.PartitionDelete) } else { return nodeDB, count, fmt.Errorf("partition doesn't exist: %s", set.PartitionDelete) diff --git a/internal/pkg/api/profile/set.go b/internal/pkg/api/profile/set.go index c35b4ade..d726b331 100644 --- a/internal/pkg/api/profile/set.go +++ b/internal/pkg/api/profile/set.go @@ -89,7 +89,7 @@ func ProfileSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodesY if set.PartitionDelete != "" { for diskname, disk := range profilePtr.Disks { if _, ok := disk.Partitions[set.PartitionDelete]; ok { - wwlog.Verbose("Node: %s, on disk %, deleting partition: %s", profileId, diskname, set.PartitionDelete) + wwlog.Verbose("Profile: %s, on disk %s, deleting partition: %s", profileId, diskname, set.PartitionDelete) delete(disk.Partitions, set.PartitionDelete) } else { return nodeDB, count, fmt.Errorf("partition doesn't exist: %s", set.PartitionDelete) @@ -99,7 +99,7 @@ func ProfileSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodesY } if set.DiskDelete != "" { if _, ok := profilePtr.Disks[set.DiskDelete]; ok { - wwlog.Verbose("Node: %s, deleting disk: %s", profileId, set.DiskDelete) + wwlog.Verbose("Profile: %s, deleting disk: %s", profileId, set.DiskDelete) delete(profilePtr.Disks, set.DiskDelete) } else { return nodeDB, count, fmt.Errorf("disk doesn't exist: %s", set.DiskDelete) @@ -107,7 +107,7 @@ func ProfileSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodesY } if set.FilesystemDelete != "" { if _, ok := profilePtr.FileSystems[set.FilesystemDelete]; ok { - wwlog.Verbose("Node: %s, deleting filesystem: %s", profileId, set.FilesystemDelete) + wwlog.Verbose("Profile: %s, deleting filesystem: %s", profileId, set.FilesystemDelete) delete(profilePtr.FileSystems, set.FilesystemDelete) } else { return nodeDB, count, fmt.Errorf("disk doesn't exist: %s", set.FilesystemDelete) diff --git a/internal/pkg/image/build.go b/internal/pkg/image/build.go index a7b49731..116b6301 100644 --- a/internal/pkg/image/build.go +++ b/internal/pkg/image/build.go @@ -34,7 +34,7 @@ func Build(name string, buildForce bool) error { var err error ignore, err = util.ReadFile(excludes_file) if err != nil { - return fmt.Errorf("failed creating directory: %s: %w", imagePath, err) + return fmt.Errorf("failed reading excludes file: %s: %w", excludes_file, err) } } diff --git a/internal/pkg/node/flags.go b/internal/pkg/node/flags.go index a1326197..21eef10c 100644 --- a/internal/pkg/node/flags.go +++ b/internal/pkg/node/flags.go @@ -58,7 +58,7 @@ func (f *boolPtrFlag) Type() string { } type NodeConfDel struct { - TagsDel []string `lopt:"tagdel" comment:"add tags"` + TagsDel []string `lopt:"tagdel" comment:"delete tags"` IpmiTagsDel []string `lopt:"ipmitagdel" comment:"delete ipmi tags"` NetTagsDel []string `lopt:"nettagdel" comment:"delete network tags"` NetDel string `lopt:"netdel" comment:"network to delete"` diff --git a/internal/pkg/warewulfd/api/image.go b/internal/pkg/warewulfd/api/image.go index 4156f196..46836196 100644 --- a/internal/pkg/warewulfd/api/image.go +++ b/internal/pkg/warewulfd/api/image.go @@ -54,7 +54,7 @@ func getImages() usecase.Interactor { } }) u.SetTitle("Get images") - u.SetDescription("Get all node images") + u.SetDescription("Get all OS images") u.SetTags("Image") return u } @@ -74,7 +74,7 @@ func getImageByName() usecase.Interactor { } }) u.SetTitle("Get an image") - u.SetDescription("Get a node image by its name") + u.SetDescription("Get an OS image by its name") u.SetTags("Image") return u } @@ -110,7 +110,7 @@ func importImage() usecase.Interactor { } }) u.SetTitle("Import an image") - u.SetDescription("Import a node image from an OCI registry") + u.SetDescription("Import an OS image from an OCI registry") u.SetTags("Image") return u @@ -146,7 +146,7 @@ func deleteImage() usecase.Interactor { return image_api.ImageDelete(cdp) }) u.SetTitle("Delete an image") - u.SetDescription("Delete an existing node image") + u.SetDescription("Delete an existing OS image") u.SetTags("Image") return u @@ -156,7 +156,7 @@ func updateImage() usecase.Interactor { type renameImageInput struct { Name string `path:"name" required:"true" description:"Name of image to update"` NewName string `json:"name" description:"New name to rename the image to"` - Build bool `query:"build" default:"true" description:"Build the image image after renaming, default:'true'"` + Build bool `query:"build" default:"true" description:"Build the OS image after renaming, default:'true'"` } u := usecase.NewInteractor(func(ctx context.Context, input renameImageInput, output *Image) error { @@ -179,7 +179,7 @@ func updateImage() usecase.Interactor { return nil }) u.SetTitle("Update or rename an image") - u.SetDescription("Update or rename an existing node image") + u.SetDescription("Update or rename an existing OS image") u.SetTags("Image") return u @@ -188,7 +188,7 @@ func updateImage() usecase.Interactor { func buildImage() usecase.Interactor { type buildImageInput struct { Name string `path:"name" required:"true" description:"Name of image to build"` - Force bool `query:"force" default:"false" description:"Build the image image even if it appears unnecessary, default:'false'"` + Force bool `query:"force" default:"false" description:"Build the OS image even if it appears unnecessary, default:'false'"` } u := usecase.NewInteractor(func(ctx context.Context, input buildImageInput, output *Image) error { @@ -206,7 +206,7 @@ func buildImage() usecase.Interactor { return nil }) u.SetTitle("Build an image") - u.SetDescription("Build a node image") + u.SetDescription("Build an OS image") u.SetTags("Image") return u diff --git a/internal/pkg/warewulfd/api/overlay.go b/internal/pkg/warewulfd/api/overlay.go index 1e09067e..656bba6d 100644 --- a/internal/pkg/warewulfd/api/overlay.go +++ b/internal/pkg/warewulfd/api/overlay.go @@ -101,13 +101,13 @@ func (of *OverlayFile) readContents() (string, error) { fullPath := of.FullPath() f, err := os.ReadFile(fullPath) if err != nil { - wwlog.Warn("os.ReadFile err %w", err) + wwlog.Warn("os.ReadFile err %v", err) return "", err } // Populate the permissions, uid, and gid. s, err := os.Stat(fullPath) if err != nil { - wwlog.Warn("os.Stat err %w", err) + wwlog.Warn("os.Stat err %v", err) return "", err } fileMode := s.Mode() diff --git a/internal/pkg/warewulfd/daemon.go b/internal/pkg/warewulfd/daemon.go index 65bfe4d7..5b4f5d49 100644 --- a/internal/pkg/warewulfd/daemon.go +++ b/internal/pkg/warewulfd/daemon.go @@ -82,7 +82,7 @@ func DaemonStatus() error { } else { err := process.Signal(syscall.Signal(0)) if err != nil { - return fmt.Errorf("failed to send process SIGCONT: %w", err) + return fmt.Errorf("failed to send signal 0 (process existence check): %w", err) } else { wwlog.Serv("Warewulf server is running at PID: %d", pid) } diff --git a/internal/pkg/warewulfd/efi.go b/internal/pkg/warewulfd/efi.go index b853d890..54edfe24 100644 --- a/internal/pkg/warewulfd/efi.go +++ b/internal/pkg/warewulfd/efi.go @@ -6,6 +6,7 @@ import ( "path" "github.com/warewulf/warewulf/internal/pkg/image" + "github.com/warewulf/warewulf/internal/pkg/util" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) @@ -38,20 +39,20 @@ func HandleEfiBoot(w http.ResponseWriter, req *http.Request) { case "grub.efi", "grub-tpm.efi", "grubx64.efi", "grubia32.efi", "grubaa64.efi", "grubarm.efi": stageFile = image.GrubFind(imageName) if stageFile == "" { - wwlog.Error("could't find grub*.efi for %s", imageName) + wwlog.Error("couldn't find grub*.efi for %s", imageName) w.WriteHeader(http.StatusNotFound) return } case "grub.cfg": stageFile = path.Join(ctx.conf.Paths.Sysconfdir, "warewulf/grub/grub.cfg.ww") tmplData = buildTemplateVars(ctx.conf, ctx.rinfo, ctx.remoteNode) - if stageFile == "" { - wwlog.Error("could't find grub.cfg template for %s", imageName) + if !util.IsFile(stageFile) { + wwlog.Error("couldn't find grub.cfg template for %s", imageName) w.WriteHeader(http.StatusNotFound) return } default: - wwlog.ErrorExc(fmt.Errorf("could't find efiboot file: %s", ctx.rinfo.efifile), "") + wwlog.ErrorExc(fmt.Errorf("couldn't find efiboot file: %s", ctx.rinfo.efifile), "") w.WriteHeader(http.StatusNotFound) return } diff --git a/internal/pkg/warewulfd/server/server.go b/internal/pkg/warewulfd/server/server.go index bca975bb..596ee5fc 100644 --- a/internal/pkg/warewulfd/server/server.go +++ b/internal/pkg/warewulfd/server/server.go @@ -97,7 +97,7 @@ func RunServer() error { auth := warewulfconf.NewAuthentication() if util.IsFile(conf.Paths.AuthenticationConf()) { if err := auth.Read(conf.Paths.AuthenticationConf()); err != nil { - wwlog.Warn("%w\n", err) + wwlog.Warn("%v", err) } } diff --git a/internal/pkg/wwlog/wwlog.go b/internal/pkg/wwlog/wwlog.go index ddb0bf60..1471fe8d 100644 --- a/internal/pkg/wwlog/wwlog.go +++ b/internal/pkg/wwlog/wwlog.go @@ -167,10 +167,10 @@ func SetLogWriterInfo(newOut io.Writer) { } /* -Set the log ofr info only +Set the log writer for errors */ func SetLogWriterErr(newOut io.Writer) { - logOut = newOut + logErr = newOut } /* diff --git a/userdocs/contributing/contributing.rst b/userdocs/contributing/contributing.rst index 38f7d47c..9f3d0f6b 100644 --- a/userdocs/contributing/contributing.rst +++ b/userdocs/contributing/contributing.rst @@ -9,7 +9,7 @@ contributing code are all ways to help! Join the community ================== -Whether you develop Warewulf or use it to deploy clusters, we hope ypu'll spread +Whether you develop Warewulf or use it to deploy clusters, we hope you'll spread the word! Share your experiences online. Ask your distribution to include support for Warewulf. Consider giving a talk at a conference or meetup! diff --git a/userdocs/contributing/development-environment.rst b/userdocs/contributing/development-environment.rst index e3e79468..b48c6274 100644 --- a/userdocs/contributing/development-environment.rst +++ b/userdocs/contributing/development-environment.rst @@ -4,7 +4,7 @@ Development Environment To develop and test the Warewulf server, you need a single system (typically a virtual machine) to serve as a test server deployment. To actually test -provisioning your development server also needs a dedicated network that it can +provisioning, your development server also needs a dedicated network that it can run DHCP on. This can typically be provisioned as a virtual network bridge in virtual machine software. @@ -17,7 +17,7 @@ Options include: A Warewulf development environment should likely use Rocky Linux 9 or openSUSE LEAP 15, though there are ongoing development efforts using Debian and Ubuntu as -well.) +well. Compiling Warewulf for a Development Server =========================================== @@ -80,7 +80,7 @@ More information: * `The golang testing package `_ * `Table Driven Tests `_ -* `Testift assert `_ +* `Testify assert `_ * `Warewulf testenv `_ Using a Dev Container @@ -89,10 +89,10 @@ Using a Dev Container Visual Studio Code (VSC) can utilize a Dev Container for a self-contained environment that has all the necessary tools and dependencies to build and test Warewulf. The Dev Container is based on the Rocky 9 image and is built using the -`devcontainer.json` file in the `.devcontainer` directory of the Warewulf -repository. To use this working Docker/Podman and VSC installations are +``devcontainer.json`` file in the ``.devcontainer`` directory of the Warewulf +repository. To use this, working Docker/Podman and VSC installations are required. To use the Dev Container, click the "Open a Remote Window" button on -the bottom left of the editor (`><` icon) and select "Reopen in Container". +the bottom left of the editor (``><`` icon) and select "Reopen in Container". This will build the container and open a new VSC window with the container as the development environment. @@ -100,6 +100,6 @@ Using Vagrant and Libvirt ========================= Vagrant can be used to quickly spin up a test/development environment for Warewulf. -A `Vagrantfile` is provided in `vagrant` directory of the Warewulf repository. +A ``Vagrantfile`` is provided in the ``vagrant`` directory of the Warewulf repository. See the `README.md `_ for more details. \ No newline at end of file diff --git a/userdocs/getting-started/debian-quickstart.rst b/userdocs/getting-started/debian-quickstart.rst index 28c538d4..64f6aebe 100644 --- a/userdocs/getting-started/debian-quickstart.rst +++ b/userdocs/getting-started/debian-quickstart.rst @@ -14,7 +14,7 @@ Install the basic services .. note:: If you get an error message concerning *isc-dhcp-server.service* you - probably need to configure the network intarface that isc-dhcp-server + probably need to configure the network interface that isc-dhcp-server will listen to. Run ``sudo dpkg-reconfigure isc-dhcp-server`` and enter the name of your cluster's private network interface (e.g. enp2s0). After that, you might also need to run ``sudo systemctl enable isc-dhcp-server``. @@ -104,8 +104,8 @@ Configure system services automatically There are a number of services and configurations that Warewulf relies on to operate. If you wish to configure all services, you can do so -individually (omitting the ``--all``) will print a help and usage -instructions. +with the following command. Running ``wwctl configure`` individually +(omitting the ``--all``) will print help and usage instructions. .. code-block:: bash diff --git a/userdocs/getting-started/el-quickstart.rst b/userdocs/getting-started/el-quickstart.rst index 0ebbafbf..d3e06ce7 100644 --- a/userdocs/getting-started/el-quickstart.rst +++ b/userdocs/getting-started/el-quickstart.rst @@ -9,7 +9,7 @@ Install Warewulf ================ The preferred way to install Warewulf on Enterprise Linux is using the -the RPMs published in `GitHub releases`_. For example, to install the +RPMs published in `GitHub releases`_. For example, to install the v4.6.5 release on Enterprise Linux 9: .. code-block:: bash @@ -153,12 +153,12 @@ configure --all``. If you just installed the system fresh and have SELinux enforcing, you may need to run ``restorecon -Rv /var/lib/tftpboot/`` to label - files written to q`tftpboot``. + files written to ``tftpboot``. -Add a base node image -===================== +Add an OS image +=============== -This will pull a basic node image from Docker Hub +This will pull a basic OS image from Docker Hub and set it for the "default" node profile. .. code-block:: bash diff --git a/userdocs/getting-started/feature-status.rst b/userdocs/getting-started/feature-status.rst new file mode 100644 index 00000000..fede28bd --- /dev/null +++ b/userdocs/getting-started/feature-status.rst @@ -0,0 +1,72 @@ +============== +Feature Status +============== + +The following table summarizes the maturity of Warewulf's top-level features. +Features are classified as follows: + +* **Stable** -- Production-ready; the interface is well-tested and unlikely to + change in incompatible ways. +* **Preview** -- Functional, but may have rough edges, limited testing, or an + interface that is still evolving. +* **Incomplete** -- Work-in-progress; key functionality or the user-facing + interface is not yet complete. + +.. list-table:: + :widths: 30 15 55 + :header-rows: 1 + + * - Feature + - Status + - Notes + * - iPXE + - Stable + - Default network bootloader used for PXE booting cluster nodes. + * - GRUB + - Preview + - Alternative bootloader; useful for UEFI and Secure Boot scenarios. + * - Single-stage boot + - Stable + - Default provisioning mode; the OS image is used directly as the root + filesystem. + * - Two-stage boot + - Preview + - Uses an intermediate initrd to initialize hardware before loading the + final OS image. Requires Dracut-based images. + * - Nodes + - Stable + - Core node management (add, remove, configure, list). + * - Profiles + - Stable + - Abstract node profiles for sharing configuration across groups of nodes. + * - Overlays + - Stable + - Template-based file provisioning applied to OS images at boot and at + runtime. + * - OS images + - Stable + - Container-based OS image import, management, and provisioning. + * - Read-only images + - Preview + - Marking an image read-only to enable future support for image + subscriptions and updates. + * - Kernels + - Stable + - Kernel extraction from OS images and per-node kernel management. + * - Disk provisioning + - Preview + - Partitioning and formatting disks on cluster nodes during provisioning. + * - Provision-to-disk + - Preview + - Provisioning an OS image to disk so that the node can subsequently boot + without Warewulf. + * - Resources + - Incomplete + - Generic node resources; the user-facing interface is not yet complete. + * - Secure Boot + - Preview + - UEFI Secure Boot support via signed bootloaders. + * - dnsmasq + - Preview + - Using dnsmasq as an alternative to ISC dhcpd and TFTP for DHCP and + network boot services. diff --git a/userdocs/getting-started/glossary.rst b/userdocs/getting-started/glossary.rst index 3d693228..bcdbdac9 100644 --- a/userdocs/getting-started/glossary.rst +++ b/userdocs/getting-started/glossary.rst @@ -9,16 +9,10 @@ Cluster network External services The Warewulf server can configure external services to support the provisioning process. For example, the Warewulf server typically deploys and - configures a DHCP server (either ISC DHCP or dnsqmasq) and a TFTP server. + configures a DHCP server (either ISC DHCP or dnsmasq) and a TFTP server. Image - The node images that Warewulf manages and provisions. Images may be imported - from OCI image registries, OCI image archives, Apptainer sandboxes, and - manual chroot directories. - - Warewulf images are maintained as an uncompressed "virtual node file system" - (VNFS, sometimes also referred to as a "chroot"). These virtual file systems - are then built as single-file images which may be used to provision a node. + See "Operating System (OS) Image" or "Overlay Image." Kernel In addition to an image, Warewulf also requires a kernel (typically a Linux @@ -42,6 +36,14 @@ nodes.conf This file is sometimes referred to as the "nodes database" or "node registry." +Operating System (OS) Image + An operating system image (or OS image) contains a bootable operating + system. When provisioning a cluster node, Warewulf combines the OS image + with the node's overlay images to provision the complete, configured image. + + OS images may be imported from OCI image registries, OCI image archives, + Apptainer sandboxes, and manual chroot directories. + Overlay Warewulf overlays provide customization for the provisioned image. Overlays may be configured on nodes or profiles, as either **system** or **runtime** @@ -56,6 +58,15 @@ Overlay Warewulf includes a number of **distribution overlays**; but additional **site overlays** can be added to a Warewulf environment. +Overlay Image + An overlay image contains the rendered contents of multiple overlays for a + given cluster node. Nodes have a "system overlay" image and a "runtime + overlay" image, based on the system and runtime overlays associated with each + node. + + Overlay images are not managed directly, but are built by the ``wwctl + overlay build`` command. + Profile Warewulf profiles are abstract nodes that carry the same configuration attributes but do not provision any specific node. Warewulf nodes may then diff --git a/userdocs/getting-started/introduction.rst b/userdocs/getting-started/introduction.rst index 4e82c558..b5778309 100644 --- a/userdocs/getting-started/introduction.rst +++ b/userdocs/getting-started/introduction.rst @@ -9,13 +9,13 @@ Warewulf is known for its massive scalability and simple management of stateless (disk optional) provisioning. Warewulf leverages a simple administrative model centralizing administration -around virtual node images which are used to provision out to the cluster nodes. -This means you can have hundreds or thousands of cluster nodes all booting and -running on the same node image. As of Warewulf v4, the node image can be managed -using industry-standard container tooling and/or CI/CD pipelines. This can be as +around virtual OS images which are used to provision cluster nodes. This means +you can have hundreds or thousands of cluster nodes all booting and running on +the same OS image. As of Warewulf v4, the OS image can be managed using +industry-standard container tooling and/or CI/CD pipelines. This can be as simple as DockerHub or your own private GitLab CI infrastructure. With this architecture, Warewulf combines the best of High Performance Computing (HPC), -Cloud, Hyperscale, and Enterprise deployment principals to create and maintain +Cloud, Hyperscale, and Enterprise deployment principles to create and maintain large scalable stateless clusters. Warewulf is used most prominently in High Performance Computing (HPC) clusters, @@ -46,8 +46,8 @@ flexible provisioning system for all types of clusters. to Dell, Debian, SUSE, Rocky, CentOS, and RHEL, Warewulf can be used in most any environment. -* **Secure**: Warewulf support SELinux out-of-the-box. Just install SELinux in - your node image and let Warewulf do the rest! +* **Secure**: Warewulf supports SELinux out-of-the-box. Just install SELinux in + your OS image and let Warewulf do the rest! * **Open Source**: Warewulf is and has always been open source. It can be used in any environment, whether public, private, non-profit, or commercial. And @@ -70,14 +70,14 @@ configuration and image and overlay assignments. **Node profiles** provide a flexible abstraction for applying configuration to multiple nodes. -**Node images** provide a bootable operating system image, including the kernel -that will be used to boot the cluster node. Node images provide a base operating +**Operating system images** provide a bootable image, including the kernel +that will be used to boot the cluster node. OS images provide the base operating system and, by default, run entirely in memory. This means that when you reboot the node, the node retains no information about Warewulf or how it booted; but it also means that they return to their initial known-good state. **Overlays** customize the provisioned operating system image with static files -and dynamic templates applied with the node image and, optionally, periodically +and dynamic templates applied with the OS image and, optionally, periodically at runtime. Beowulf overview diff --git a/userdocs/getting-started/network.rst b/userdocs/getting-started/network.rst index d824bc40..baf789eb 100644 --- a/userdocs/getting-started/network.rst +++ b/userdocs/getting-started/network.rst @@ -63,7 +63,7 @@ the management of ``dhcpd`` and ``tftp``. There are two ways to achieve this: The first method is relatively trivial. The second method is described below. -As first the first step, add the dummy node. +As the first step, add the dummy node. .. code-block:: shell @@ -87,8 +87,8 @@ Add the delivery networks to this node. --nettagadd network=10.0.30.0,dynstart=10.10.30.10,dynend=10.10.30.50 \ deliverynet -The ip address is used as the network address of host in the delivery network -and an additional tags is used for definition of the network itself and the +The IP address is used as the network address of the host in the delivery network +and an additional tag is used for the definition of the network itself and the dynamic dhcp range. You can check the result with ``wwctl node list``. .. code-block:: console diff --git a/userdocs/getting-started/provisioning.rst b/userdocs/getting-started/provisioning.rst index a0f5b7d4..cd48faba 100644 --- a/userdocs/getting-started/provisioning.rst +++ b/userdocs/getting-started/provisioning.rst @@ -10,7 +10,7 @@ version and configuration drift, eventually becoming a point of discrete administration and debugging. The larger the cluster, the harder this problem is to solve. -This is the problem that Warewulf was created solve. +This is the problem that Warewulf was created to solve. Provisioning Overview ===================== @@ -78,7 +78,7 @@ When a cluster node boots from Warewulf, the following process occurs: * In a single-stage provisioning configuration, the desired image and overlays are combined and provisioned immediately by the bootloader as the - kernel's initial root file system. This is straightfoward, but does not + kernel's initial root file system. This is straightforward, but does not work in all environments: some systems have memory layouts that are not handled properly by either iPXE or GRUB for sufficiently large image sizes, leading to strange, unpredictable results. diff --git a/userdocs/getting-started/quick-reference.rst b/userdocs/getting-started/quick-reference.rst new file mode 100644 index 00000000..d9450d8b --- /dev/null +++ b/userdocs/getting-started/quick-reference.rst @@ -0,0 +1,223 @@ +=============== +Quick Reference +=============== + +A quick reference for common day-to-day Warewulf operations. + +Important paths +=============== + +.. code-block:: text + + # Configuration + /etc/warewulf/warewulf.conf # Main server configuration + /etc/warewulf/nodes.conf # Node database + /etc/warewulf/auth.conf # API authentication + + # Images (container filesystems) + /var/lib/warewulf/chroots/ # Image root directories + + # Overlays + /usr/share/warewulf/overlays/ # Distribution-provided overlays + /var/lib/warewulf/overlays/ # Site-local overlays + /var/lib/warewulf/provision/ # Built overlay images (per-node) + + # Logs + journalctl -u warewulfd # Warewulf daemon logs + +Server management +================= + +.. code-block:: shell + + # Configure all external services (TLS, warewulfd, TFTP, DHCP, NFS, SSH, hosts) + wwctl configure --all + + # Configure individual services + wwctl configure dhcp + wwctl configure tftp + wwctl configure nfs + wwctl configure ssh + wwctl configure hostfile + + # Restart the Warewulf daemon + systemctl restart warewulfd + +Listing nodes and profiles +=========================== + +.. code-block:: shell + + # List all nodes (summary) + wwctl node list + + # List nodes with network interface details + wwctl node list --net + + # List nodes with all fields (includes unset/inherited values) + wwctl node list --all n1 + + # List nodes with IPMI settings + wwctl node list --ipmi + + # List specific nodes using hostlist syntax + wwctl node list n[1-10] + + # Show current node boot status + wwctl node status + + # List all profiles + wwctl profile list + + # List all fields of a profile (includes inherited values) + wwctl profile list default --all + +Adding a node +============= + +.. code-block:: shell + + # Add a single node with an IP address + wwctl node add n1 --ipaddr=10.0.2.1 + + # Add a range of nodes (IP address auto-increments) + wwctl node add n[2-4] --ipaddr=10.0.2.2 + + # Add a node with multiple options: image, network interface, kernel args, and profile + wwctl node add n1 \ + --ipaddr=10.0.2.1 \ + --netmask=255.255.255.0 \ + --netdev=eno1 \ + --hwaddr=00:00:00:00:00:01 \ + --image=rockylinux-9 \ + --profile=default \ + --kernelargs="quiet crashkernel=no" + + # Un-set a field (revert to profile/default value) + wwctl node set n1 --image=UNDEF + +Network interfaces +================== + +.. code-block:: shell + + # Set the primary network interface + wwctl node set n1 \ + --netdev=eno1 \ + --hwaddr=00:00:00:00:00:01 \ + --ipaddr=10.0.2.1 \ + --netmask=255.255.255.0 + + # Add a secondary network (e.g. InfiniBand) + wwctl node set n1 \ + --netname=infiniband \ + --type=infiniband \ + --netdev=ib0 \ + --ipaddr=10.0.3.1 \ + --netmask=255.255.255.0 + + # Configure a VLAN interface + wwctl node set n1 \ + --netname=vlan42 \ + --netdev=vlan42 \ + --type=vlan \ + --ipaddr=10.0.42.1 \ + --netmask=255.255.252.0 \ + --nettagadd="vlan_id=42,parent_device=eth0" + + # Set DNS on an interface + wwctl node set n1 --nettagadd="DNS1=1.1.1.1" + +Images +====== + +.. code-block:: shell + + # Import an image from Docker Hub (or any OCI registry) + wwctl image import docker://ghcr.io/warewulf/warewulf-rockylinux:9 rockylinux-9 + + # Import with registry credentials + wwctl image import \ + --username myuser --password mysecret \ + docker://registry.example.com/myimage:latest myimage + + # Import from a local directory or tarball + wwctl image import ./rockylinux-9/ rockylinux-9 + wwctl image import rockylinux-9.tar rockylinux-9 + + # List available images + wwctl image list + + # Open an interactive shell inside an image + wwctl image shell rockylinux-9 + + # Run a single command inside an image (e.g. install packages) + wwctl image exec rockylinux-9 -- dnf -y install vim htop + + # Rebuild the image archive (after making changes) + wwctl image build rockylinux-9 + + # Assign an image to a node or profile + wwctl node set n1 --image=rockylinux-9 + wwctl profile set default --image=rockylinux-9 + +IPMI +==== + +.. code-block:: shell + + # Configure IPMI settings on the default profile + wwctl profile set default \ + --ipminetmask=255.255.255.0 \ + --ipmiuser=admin \ + --ipmipass=passw0rd \ + --ipmiinterface=lanplus \ + --ipmiwrite + + # Set the IPMI address on a specific node + wwctl node set n1 --ipmiaddr=192.168.2.1 + + # List IPMI settings + wwctl node list --ipmi + + # Power commands + wwctl power status n[1-10] + wwctl power on n1 + wwctl power off n1 + wwctl power cycle n1 + wwctl power reset n1 + + # Open serial-over-LAN console + wwctl node console n1 + +Overlays +======== + +.. code-block:: shell + + # List all available overlays + wwctl overlay list + + # Build overlays for all nodes + wwctl overlay build + + # Build overlays for a specific node + wwctl overlay build n1 + + # Create a new site-local overlay + wwctl overlay create myoverlay + + # Import a file from the host into an overlay + wwctl overlay import myoverlay /etc/motd + + # Edit a file in an overlay (opens $EDITOR) + wwctl overlay edit myoverlay /etc/motd + + # Show an overlay file (optionally rendered for a specific node) + wwctl overlay show myoverlay /etc/motd + wwctl overlay show myoverlay /etc/motd.ww --render=n1 + + # Assign overlays to all nodes via the default profile + wwctl profile set default \ + --system-overlays="wwinit,wwclient,fstab,hostname,ssh.host_keys,NetworkManager" \ + --runtime-overlays="hosts,ssh.authorized_keys" diff --git a/userdocs/getting-started/suse-quickstart.rst b/userdocs/getting-started/suse-quickstart.rst index 7e95822c..0f5f9fca 100644 --- a/userdocs/getting-started/suse-quickstart.rst +++ b/userdocs/getting-started/suse-quickstart.rst @@ -42,7 +42,7 @@ Configure the controller ======================== Edit the file ``/etc/warewulf/warewulf.conf`` and ensure that you've -set the appropriate configuration paramaters. Here are some of the +set the appropriate configuration parameters. Here are some of the defaults for reference assuming that ``192.168.200.1`` is the IP address of your cluster's private network interface: @@ -97,8 +97,8 @@ Configure system services automatically There are a number of services and configurations that Warewulf relies on to operate. If you wish to configure all services, you can do so -individually (omitting the ``--all``) will print a help and usage -instructions. +with the following command. Running ``wwctl configure`` individually +(omitting the ``--all``) will print help and usage instructions. .. note:: diff --git a/userdocs/images/images.rst b/userdocs/images/images.rst index 7476b831..d2d3d09b 100644 --- a/userdocs/images/images.rst +++ b/userdocs/images/images.rst @@ -1,15 +1,15 @@ .. _images: -=========== -Node Images -=========== +========= +OS Images +========= -Warewulf node images are a "Virtual Node File System" (VNFS) that serves as a -base image for cluster nodes. This is similar to a "golden master" image, except +Warewulf operating system (OS) images are a "Virtual Node File System" (VNFS) +that serves as an image for cluster nodes. This is similar to a "golden master" image, except that the image source exists mutably within a directory on the Warewulf control node (e.g. a ``chroot()``). -Warewulf node images have several similarities to Linux containers; so Warewulf +Warewulf OS images have several similarities to Linux containers; so Warewulf v4 integrates directly within the container ecosystem to facilitate the process of image creation and image management: images can be built, for example, with Docker, Podman, or Apptainer, and imported directly from OCI registries or local @@ -58,7 +58,7 @@ or Apptainer sandbox. OCI Registry ------------ -You can import node images from an OCI registry, public or private. +You can import OS images from an OCI registry, public or private. .. code-block:: console @@ -76,13 +76,13 @@ You can import node images from an OCI registry, public or private. Most images in Docker Hub are not "bootable": they typically do not include a kernel, and likely don't include any init system. For this reason, don't - expect a base image from DockerHub (e.g. ``docker://rockylinux`` or + expect just any image from DockerHub (e.g. ``docker://rockylinux`` or ``docker://debian``) to boot properly with Warewulf. - The Warewulf project maintains a set of `example node images + The Warewulf project maintains a set of `example OS images `_ that are configured to boot when used with Warewulf. These images can be imported directly into - Warewulf or used as base images for local custom image. + Warewulf or used as base images for local custom images. A few environmental variables can be used to control communication with the OCI registry: @@ -118,7 +118,7 @@ See ProxyFromEnvironment_ for more information. The above is just an example. Consideration should be done before doing it this way if you are in a security sensitive environment or shared environments as -this command line wil show up in the process table. +this command line will show up in the process table. Local OCI Archive ----------------- @@ -300,7 +300,7 @@ Exit Script Warewulf executes the script ``/etc/warewulf/image_exit.sh`` in the image after a ``wwctl image shell`` or ``wwctl image exec`` and prior to (re)building the -final node image for delivery. This is typically used to remove cache or log +final OS image for delivery. This is typically used to remove cache or log files that may have been generated by the executed command or interactive session. @@ -310,10 +310,10 @@ package repository caches that may have been generated. Defining New Images =================== -It is absolutely possible to import a base image into Warewulf and make all +It is absolutely possible to import an OS image into Warewulf and make all changes interactively with ``wwctl image shell``; but it is often better to define new images with a container image definition file. This can be done using -the OCI and Singularity (Apptainer) ecoystems. +the OCI and Singularity (Apptainer) ecosystems. Podman ------ @@ -345,7 +345,7 @@ particularly easy to do with Apptainer. .. _OCI base image: https://docs.docker.com/build/building/base-images/ -Consider the following file called `warewulf-rockylinux-9.def`: +Consider the following file called ``warewulf-rockylinux-9.def``: .. code-block:: singularity @@ -427,7 +427,7 @@ Image Architecture By default, Warewulf will try to import an image of the same platform (e.g., amd64, arm64) as the local system. To specify the platform to import, either -specify `WAREWULF_OCI_PLATFORM` or use the argument `--platform` during import. +specify ``WAREWULF_OCI_PLATFORM`` or use the argument ``--platform`` during import. It is possible to build, edit, and provision images of different architectures (i.e. aarch64) from an x86_64 host by using QEMU. Simply run the appropriate diff --git a/userdocs/images/selinux.rst b/userdocs/images/selinux.rst index 981a24f7..2f592f16 100644 --- a/userdocs/images/selinux.rst +++ b/userdocs/images/selinux.rst @@ -5,8 +5,8 @@ SELinux-enabled Images ====================== Warewulf supports booting SELinux-enabled images, though nodes using SELinux -must be configured to use tmpfs for their image file system. ("ramfs," often -used by default, does not support extended file attributes, which are required +must be configured to use tmpfs for their image file system. (The default +``initramfs`` root does not support extended file attributes, which are required for SELinux context labeling.) .. code-block:: bash @@ -16,8 +16,8 @@ for SELinux context labeling.) .. note:: Versions of Warewulf prior to v4.5.8 also required a kernel argument - "rootfstype=ramfs" in order for wwinit to copy the node image to tmpfs; but - this is no longer required. + "rootfstype=ramfs" in order for wwinit to copy the image to tmpfs; but this + is no longer required. Once that is done, enable SELinux in ``/etc/sysconfig/selinux`` and install the appropriate packages in the image. `An example`_ of such an image is available @@ -26,4 +26,4 @@ in the warewulf-node-images repository. .. _An example: https://github.com/warewulf/warewulf-node-images/tree/main/examples/rockylinux-9-selinux SELinux requires extended attributes, which aren't supported on a default -``initrootfs``. Nodes using SELinux should specify ``--root=tmpfs``. \ No newline at end of file +``initramfs`` root. Nodes using SELinux should specify ``--root=tmpfs``. \ No newline at end of file diff --git a/userdocs/images/syncuser.rst b/userdocs/images/syncuser.rst index 8ab9fa13..569237b3 100644 --- a/userdocs/images/syncuser.rst +++ b/userdocs/images/syncuser.rst @@ -2,21 +2,37 @@ Syncuser ======== -Warewulf can optionally synchronize UIDs and GIDs from the Warewulf server to an -image. This can be particularly useful when there is no central directory (e.g., -an LDAP server). +Warewulf's syncuser feature has two distinct parts that work together: + +1. The **syncuser command** synchronizes UIDs and GIDs from the Warewulf server + into an OS image, updating ``/etc/passwd``, ``/etc/group``, and file + ownerships within the image. + +2. The **syncuser overlay** merges users and groups from both the OS image and + the Warewulf server into the provisioned node's ``/etc/passwd`` and + ``/etc/group`` at runtime. + +This is particularly useful when there is no central directory (e.g., an LDAP +server). Together, these two parts ensure that UIDs and GIDs are consistent +across the server and all cluster nodes. .. note:: - - Some system services (notably "munge") require a user to have the same UID across all nodes. -Combined with the "syncuser" overlay, Warewulf syncuser also supports defining -local users on the Warewulf server for synchronization to cluster nodes. + Some system services (notably "munge") require a user to have the same UID + across all nodes. -If there is mismatch between the server and the image, the import command will -generate a warning. +Synchronizing UIDs/GIDs in an OS image +-------------------------------------- -Syncuser may be invoked during image import, exec, shell, or build. +The syncuser command updates an OS image so that any user or group present on +the Warewulf server has the same UID/GID inside the image. Users and groups +that exist only in the image are preserved unless a UID/GID collision is +detected. File ownerships within the image are also updated to match. + +If there is a mismatch between the server and the image, the import command +will generate a warning. + +Syncuser may be invoked during image import, exec, shell, or build: .. code-block:: console @@ -26,5 +42,18 @@ Syncuser may be invoked during image import, exec, shell, or build. # wwctl image build --syncuser rockylinux-9 # wwctl image syncuser rockylinux-9 -After syncuser, ``/etc/passwd`` and ``/etc/group`` in the image are updated, and -permissions on files belonging to these UIDs and GIDs are updated to match. +After syncuser, ``/etc/passwd`` and ``/etc/group`` in the image are updated, +and permissions on files belonging to these UIDs and GIDs are updated to match. + +The syncuser overlay +-------------------- + +The syncuser overlay runs at provisioning time and merges ``/etc/passwd`` and +``/etc/group`` from both the OS image and the Warewulf server. This makes users +defined on the server (but not originally in the image) available on provisioned +nodes. + +For the overlay to work correctly, the image should also have been prepared with +the syncuser command (see above) so that UID/GID values are consistent. + +See :ref:`Syncuser` in the overlays documentation for more detail. diff --git a/userdocs/index.rst b/userdocs/index.rst index 6558330c..7b0db5da 100644 --- a/userdocs/index.rst +++ b/userdocs/index.rst @@ -15,6 +15,8 @@ Welcome to the Warewulf User Guide! SUSE Quickstart (openSUSE Leap and SLES) Debian Quickstart Glossary + Quick Reference + Feature Status .. toctree:: :maxdepth: 1 @@ -44,7 +46,7 @@ Welcome to the Warewulf User Guide! :maxdepth: 1 :caption: Images - Node Images + OS Images Image Kernels Syncuser SELinux diff --git a/userdocs/nodes/disks.rst b/userdocs/nodes/disks.rst index 2e14e3a3..04cb2abf 100644 --- a/userdocs/nodes/disks.rst +++ b/userdocs/nodes/disks.rst @@ -12,11 +12,6 @@ sfdisk, mkfs, and mkswap during boot. Ignition can, for example, create ``swap`` partitions or ``/scratch`` file systems. -.. note:: - - Warewulf is not currently able to provision the node image onto an explicitly - provisioned root file system. - Requirements ============ @@ -55,7 +50,7 @@ sizes should also be set (specified in MiB), except for the last partition: if no size is given, the maximum available size is used. Each partition has the switches ``should_exist`` and ``wipe_partition_entry`` which control the partition creation process (via the ``--partcreate`` and ``--partwipe`` flags). -When omitting a partition number the `wipe_partition_entry` should be true, as +When omitting a partition number the ``wipe_partition_entry`` should be true, as this allows ignition to replace the existing partition. .. code-block:: shell @@ -103,7 +98,7 @@ by the ``ignition`` overlay. Example disk configurations =========================== -This command formats a btrfs file system on a "scratch" partion of +This command formats a btrfs file system on a "scratch" partition of "vda" and mounts it at ``/scratch``. .. code-block:: shell @@ -136,10 +131,10 @@ disk, as necessary. If you would like to re-use existing partitions but want to replace existing file systems once, you may -* wipe the existing data with tools like ``wipefs`` or `dd` [#]_; or +* wipe the existing data with tools like ``wipefs`` or ``dd`` [#]_; or * set the ``--fswipe`` flag and remove it after one reboot. -.. [#] With ``wipefs`` you have to remove the filesystem *and* parition +.. [#] With ``wipefs`` you have to remove the filesystem *and* partition information. E.g., use ``wipefs -fa /dev/vda*`` to remove all file system information and partition information. @@ -148,6 +143,224 @@ See the `upstream ignition documentation`_ for additional information. .. _upstream ignition documentation: https://coreos.github.io/ignition/operator-notes/#filesystem-reuse-semantics +.. _swap-and-image-memory: + +Swap and image memory usage +=========================== + +Warewulf images run entirely in memory. Configuring a local swap partition +can allow the kernel to reclaim that RAM for applications — but only under the +right conditions. Whether swap can free image memory depends on which root +filesystem type the node uses. + +tmpfs root +---------- + +When the root filesystem is ``tmpfs`` (the default for two-stage dracut boot, or +when ``--root=tmpfs`` is set explicitly), the image lives in the page cache. +The Linux kernel can swap ``tmpfs`` pages to a local swap device exactly as it +would any other anonymous memory. Adding swap therefore lets the kernel evict +cold image pages to disk and reclaim that RAM for running workloads. + +This is the recommended configuration for nodes with large images relative to +available RAM. + +initramfs root (single-stage boot default) +------------------------------------------ + +The default single-stage boot places the image in an ``initramfs`` root, which +is an instance of ``ramfs``. Unlike ``tmpfs``, ``ramfs`` pages are pinned in +memory: the kernel will never swap them out. Configuring swap on a node with an +``initramfs`` root will **not** free any memory used by the image. + +If you are using single-stage boot and want swap to help with image memory, +switch to ``tmpfs`` root first: + +.. code-block:: shell + + wwctl profile set default --root=tmpfs + +For background on ``tmpfs`` NUMA interleaving and size limits, see +:ref:`tmpfs-and-numa`. + +.. note:: + + Two-stage dracut boot always uses ``tmpfs`` regardless of the ``--root`` + setting. If you are already using dracut, no additional root configuration + is needed. + +Example: configuring swap to reclaim image memory +-------------------------------------------------- + +This example provisions a swap partition on the local disk of each node and +activates it at boot, enabling the kernel to reclaim RAM occupied by the node +image. + +**1. Configure tmpfs root** (skip if using two-stage dracut boot): + +.. code-block:: shell + + wwctl profile set default --root=tmpfs + +**2. Add a swap partition** to the node or profile disk configuration. This +example adds an 8 GiB swap partition as the first partition of ``/dev/vda``: + +.. code-block:: shell + + wwctl profile set default \ + --diskname /dev/vda --diskwipe \ + --partname swap --partcreate --partnumber 1 --partsize=8192 \ + --fsname swap --fsformat swap --fspath swap + +**3. Add the required overlays** to the system overlay so that the swap +partition is formatted and activated at boot. The ``-O`` / ``--system-overlays`` +flag replaces the entire overlay list, so include any overlays already +configured for the profile. Use Ignition (recommended for Rocky Linux 9 and +openSUSE): + +.. code-block:: shell + + wwctl profile set default \ + -O wwinit,wwclient,fstab,hostname,ssh.host_keys,systemd.netname,NetworkManager,ignition,systemd.swap + +Or, for systems without Ignition (e.g., Rocky Linux 8), use the ``sfdisk`` and +``mkswap`` overlays instead: + +.. code-block:: shell + + wwctl profile set default \ + -O wwinit,wwclient,fstab,hostname,ssh.host_keys,systemd.netname,NetworkManager,sfdisk,mkswap,systemd.swap + +**4. Rebuild the dracut initramfs** with the tools needed to provision the disk +during the two-stage boot (skip for single-stage boot): + +.. code-block:: shell + + # Ignition path + wwctl image exec rockylinux-9 -- /usr/bin/dracut --force --no-hostonly \ + --add wwinit --add ignition --regenerate-all + + # sfdisk/mkswap path + wwctl image exec rockylinux-8 -- /usr/bin/dracut --force --no-hostonly \ + --add wwinit --install sfdisk --install blockdev --install udevadm \ + --install mkswap --regenerate-all + +**5. Reboot nodes** to apply the new configuration. + +Verifying swap is active +------------------------ + +After reboot, confirm that the swap device is active: + +.. code-block:: shell + + swapon --show + +Then check the overall memory picture with ``free -h``: + +.. code-block:: text + + $ free -h + total used free shared buff/cache available + Mem: 15Gi 12Gi 256Mi 120Mi 2.8Gi 2.8Gi + Swap: 8.0Gi 0Ki 8.0Gi + +At idle, most of the node's RAM is occupied by the image. The ``Swap:`` line +shows the available device but near-zero usage at this point. + +Confirming that the image gets swapped out +------------------------------------------ + +To demonstrate that the kernel evicts image pages to swap when an application +needs memory, first note the image size with ``df -h /``: + +.. code-block:: shell + + df -h / + +This shows how much tmpfs space the image occupies — that is the amount of RAM +currently holding the image. + +Now apply memory pressure using ``stress-ng`` (install it in the OS image if +not already present). The allocation must exceed **available** RAM — not just +total RAM — to force the kernel to evict image pages. Compute the target from +``MemTotal``: + +.. code-block:: shell + + stress-ng --vm 1 \ + --vm-bytes $(awk '/MemTotal/{printf "%dM", $2*0.9/1024}' /proc/meminfo) \ + --vm-keep -t 60s & + +While ``stress-ng`` is running, observe memory usage: + +.. code-block:: shell + + free -h + +.. code-block:: text + + $ free -h + total used free shared buff/cache available + Mem: 15Gi 15Gi 32Mi 120Mi 512Mi 192Mi + Swap: 8.0Gi 4.2Gi 3.8Gi + +The ``Swap: used`` value has grown by roughly the size of the image. The +kernel has evicted cold image pages to swap, making that RAM available to the +application. The application can access the full physical memory of the node, +not just what is left over after the image is loaded. + +After ``stress-ng`` finishes, the evicted image pages remain on swap until +they are accessed again, so ``free -h`` will continue to show swap usage until +the node is under less pressure and pages are faulted back in as needed. + +Moving image pages to swap proactively +-------------------------------------- + +Rather than simulating a workload, you can instruct the kernel to push image +pages to swap directly. On Linux 6.1 and later with cgroup v2, write the +desired reclaim size to ``memory.reclaim``: + +.. code-block:: shell + + # Request that the kernel reclaim up to 4 GiB from the root cgroup + # (adjust to match your image size shown by df -h /) + echo "4G" > /sys/fs/cgroup/memory.reclaim + +If the kernel cannot reclaim the full requested amount it returns an error: + +.. code-block:: text + + -bash: echo: write error: Resource temporarily unavailable + +This is expected and benign. It means the kernel reclaimed as much as it could +but the reclaimable pool was exhausted before reaching the target. + +The kernel will swap out reclaimable pages until the target is met or exhausted, then stop. +Run ``free -h`` immediately afterwards to see the reduction in ``Mem: used`` +and corresponding increase in ``Swap: used``. + +This works best when run early in the boot process, before applications start, +when nearly all anonymous memory belongs to the image. To automate it, add a +systemd unit to a custom overlay that runs at ``local-fs.target``: + +.. code-block:: ini + + [Unit] + Description=Reclaim OS image memory to swap + After=local-fs.target + ConditionPathExists=/sys/fs/cgroup/memory.reclaim + + [Service] + Type=oneshot + ExecStart=/bin/sh -c 'echo "$(df --output=used / | tail -1)K" > /sys/fs/cgroup/memory.reclaim' + + [Install] + WantedBy=multi-user.target + +This reads the actual image size from ``df`` and requests that exact amount be +reclaimed, so it adapts automatically as the image grows or shrinks. + .. _provision to disk: Provision to disk @@ -254,7 +467,7 @@ functionality is used: Configuring the root device --------------------------- -Set the desired storage device for the node image using the ``--root`` +Set the desired storage device for the image using the ``--root`` parameter. .. code-block:: shell diff --git a/userdocs/nodes/ipmi.rst b/userdocs/nodes/ipmi.rst index 8759bfa7..7eec1f76 100644 --- a/userdocs/nodes/ipmi.rst +++ b/userdocs/nodes/ipmi.rst @@ -13,7 +13,7 @@ Configuration Typically, common settings for IPMI interfaces are set on a profile, leaving only the IP address set per-node. -If ``--ipmiwrite`` is set to `true`, the ``wwinit`` overlay will write the +If ``--ipmiwrite`` is set to ``true``, the ``wwinit`` overlay will write the desired IPMI configuration to the node's BMC during boot. .. code-block:: diff --git a/userdocs/nodes/nodes.rst b/userdocs/nodes/nodes.rst index 06d895ba..42e40f2a 100644 --- a/userdocs/nodes/nodes.rst +++ b/userdocs/nodes/nodes.rst @@ -8,8 +8,8 @@ supported; but it is often better to manage it using the ``wwctl`` command. .. note:: - The ``nodes.conf`` file is YAML document that can be edited directly or - managed with configuation management; but its internal structure is + The ``nodes.conf`` file is a YAML document that can be edited directly or + managed with configuration management; but its internal structure is technically undocumented and subject to change between versions. After Warewulf v4.6.0, the ``wwctl upgrade nodes`` command can be used to update a ``nodes.conf`` from a previous Warewulf v4 version. @@ -99,7 +99,7 @@ edit``. List values ----------- -Some node fields, such as overlays and kernel aruments, accept a list of values. +Some node fields, such as overlays and kernel arguments, accept a list of values. These may be specified as a comma-separated list or as multiple arguments. To include an explicit comma in the value, enclose the value in inner-quotes. @@ -113,7 +113,8 @@ To include an explicit comma in the value, enclose the value in inner-quotes. Un-setting Node Fields ---------------------- -To un-set a field value, set the value to ``UNDEF``. +To un-set a field value, set the value to ``UNDEF`` or ``UNSET`` (both are +accepted). .. code-block:: shell diff --git a/userdocs/overlays/overlays.rst b/userdocs/overlays/overlays.rst index 23bca8e7..9b497389 100644 --- a/userdocs/overlays/overlays.rst +++ b/userdocs/overlays/overlays.rst @@ -2,12 +2,12 @@ Overlays ======== -Warewulf supplements provisioned node images with an "overlay" system. Overlays +Warewulf supplements provisioned OS images with an "overlay" system. Overlays are collections of files and :ref:`templates` that are rendered and built per-node and then applied over the image during the provisioning process. -Overlays are the primary mechanism for adding functionality Warewulf. Much of +Overlays are the primary mechanism for adding functionality to Warewulf. Much of even core functionality in Warewulf is implemented as distribution overlays, and this flexibility is also available for local, custom overlays. By combining templates with tags, network tags, and resources, the node registry @@ -84,7 +84,7 @@ A node or profile can configure an overlay in two different ways: wwctl profile set default \ --system-overlays="wwinit,wwclient,fstab,hostname,ssh.host_keys,systemd.netname,NetworkManager" \ - --runime-overlays="hosts,ssh.authorized_keys" + --runtime-overlays="hosts,ssh.authorized_keys" Multiple overlays can be applied to a single node, and overlays from multiple profiles are appended together when applied to a single node. @@ -92,7 +92,7 @@ profiles are appended together when applied to a single node. Building Overlays ================= -Overlays are built (e.g., with ``wwctl overly build``) into compressed overlay +Overlays are built (e.g., with ``wwctl overlay build``) into compressed overlay images for distribution to cluster nodes. These images typically match these two use cases: system and runtime. As such, each cluster node typically has two overlay images. @@ -107,7 +107,7 @@ overlay images. Created image for n1 runtime overlay: /var/lib/warewulf/provision/overlays/n1/__RUNTIME__.img Compressed image for n1 runtime overlay: /var/lib/warewulf/provision/overlays/n1/__RUNTIME__.img.gz -Overlay images for multiple node are built in parallel. By default, each CPU in +Overlay images for multiple nodes are built in parallel. By default, each CPU in the Warewulf server will build overlays independently. The number of workers can be specified with the ``--workers`` option. @@ -239,7 +239,7 @@ provisioning actions: wwclient -------- -All configured overlays are provisioned initially along with the node image +All configured overlays are provisioned initially along with the OS image itself; but **wwclient** periodically fetches and applies the runtime overlay to allow configuration of some settings without a reboot. @@ -342,7 +342,7 @@ resource. It also creates entries for file systems defined by Ignition. ssh --- -Two SSH overlays configure host keys (one set for all node in the cluster) and +Two SSH overlays configure host keys (one set for all nodes in the cluster) and ``authorized_keys`` for the root account. - ssh.authorized_keys @@ -353,12 +353,14 @@ Two SSH overlays configure host keys (one set for all node in the cluster) and syncuser -------- -The **syncuser** overlay updates ``/etc/passwd`` and ``/etc/group`` to include -all users on both the Warewulf server and from the image. +The **syncuser** overlay updates ``/etc/passwd`` and ``/etc/group`` at +provisioning time to include all users from both the Warewulf server and from +the image. To function properly, ``wwctl image syncuser`` (or the ``--syncuser`` option -during ``import``, ``exec``, ``shell``, or ``build``) must have also been run on -the image to synchronize its user and group IDs with those of the server. +during ``import``, ``exec``, ``shell``, or ``build``) must also have been run on +the image to synchronize its user and group IDs with those of the server. See +:doc:`/images/syncuser` for details on both parts of the syncuser feature. If a ``PasswordlessRoot`` tag is set to "true", the overlay will also insert a "passwordless" root entry. This can be particularly useful for accessing a @@ -408,7 +410,7 @@ debug ----- The **debug** overlay is not intended to be used in configuration, but is -provided as an example. In particular, the provided `tstruct.md.ww` demonstrates +provided as an example. In particular, the provided ``tstruct.md.ww`` demonstrates the use of most available template metadata. .. code-block:: shell @@ -520,6 +522,10 @@ include wipefs to detect existing file systems. wwctl image exec rockylinux-9 -- /usr/bin/dracut --force --no-hostonly --add wwinit --install mkswap --regenerate-all +For a complete walkthrough of configuring swap to free memory consumed by the +in-memory image, including which root filesystem types support swap reclamation, +see :ref:`swap-and-image-memory`. + systemd mounts -------------- @@ -585,7 +591,7 @@ Once configured, the NVIDIA MIG instances can be mapped to Slurm GRES entries backed by ``/dev/nvidia-caps/capXX`` device files. To generate the mapping of MIG instances to ``/dev/nvidia-caps/capXX`` device files -for Slurm's ``gres.conf`` file, use the script provided by this overlay on on the node: +for Slurm's ``gres.conf`` file, use the script provided by this overlay on the node: .. code-block:: shell @@ -603,7 +609,7 @@ run. (Subsequent use of the host overlay won't overwrite existing The following services get configuration files via the host overlay: -* ssh keys are created with the scrips ``ssh_setup.sh`` and +* ssh keys are created with the scripts ``ssh_setup.sh`` and ``ssh_setup.csh`` * hosts entries are created by manipulating ``/etc/hosts`` with the template ``hosts.ww`` diff --git a/userdocs/overlays/templates.rst b/userdocs/overlays/templates.rst index 437a21c6..b3a69d9d 100644 --- a/userdocs/overlays/templates.rst +++ b/userdocs/overlays/templates.rst @@ -215,8 +215,8 @@ nobackup UniqueField ----------- -Returns a filtered version of a multi-line input string. input is expected to be -a field-separated format with one record per line (terminated by `\n`). Order of +Returns a filtered version of a multi-line input string. Input is expected to be +a field-separated format with one record per line (terminated by ``\n``). Order of lines is preserved, with the first matching line taking precedence. For example, the following template snippet has been used in the ``syncuser`` overlay diff --git a/userdocs/release/v4.6.0.rst b/userdocs/release/v4.6.0.rst index 377862b4..a8df7aab 100644 --- a/userdocs/release/v4.6.0.rst +++ b/userdocs/release/v4.6.0.rst @@ -20,7 +20,7 @@ well, and the reorganization and deduplication supports better documentation maintenance in the future. The documentation also now includes complete and automatically-generated -references for all ``wwctl`` commands, sub-comands, and options. +references for all ``wwctl`` commands, sub-commands, and options. .. _user documentation: https://warewulf.org/docs/ @@ -92,7 +92,7 @@ Warewulf "containers" have never been "virtualized" or executed with a container runtime. Rather, the name "container" was selected to imply the integration in v4 with the container ecosystem of tooling for defining, building, storing, and testing node images. But this terminology ended up causing persistent confusion, -so a more industry-standard "node image" terminology has been adopted in stead. +so a more industry-standard "node image" terminology has been adopted instead. The ``wwctl container`` command is retained as an alias for the new ``wwctl image`` command. The variables ``.Container`` and ``.ContainerName`` are also @@ -102,7 +102,7 @@ will continue to work through the v4.6.x series. There are smaller changes to the image system, as well: - ``wwctl image shell`` now supports a ``WW_HISTFILE`` environment variable to - save shell history _inside_ the image. + save shell history *inside* the image. - ``wwctl image shell`` now supports a ``WW_PS1`` environment variable to specify the prompt for the interactive shell. The default prompt has also been updated to indicate the current directory. @@ -228,7 +228,7 @@ Profiles ======== Node profiles now support profiles themselves, allowing for complex nested -hierarchies of nested profiles. +hierarchies of profiles. .. code-block:: yaml @@ -363,7 +363,7 @@ expressions and globs) are no longer supported. in YAML and JSON format, and ``wwctl node export`` has been updated to match, including indicating node IDs. -``wwctl`` now return a non-zero exit code on error. +``wwctl`` now returns a non-zero exit code on error. There are smaller changes to ``wwctl``, as well: diff --git a/userdocs/release/v4.6.1.rst b/userdocs/release/v4.6.1.rst index 450af0e1..5932a465 100644 --- a/userdocs/release/v4.6.1.rst +++ b/userdocs/release/v4.6.1.rst @@ -36,7 +36,7 @@ A couple new ``wwctl`` options have been added: - ``wwctl overlay import --overwrite`` overwrites existing overlay files during import. -- ``wwctl node import --yes`` skips the interactive confirmaton dialog and +- ``wwctl node import --yes`` skips the interactive confirmation dialog and assumes a "yes" answer. New tags @@ -84,7 +84,7 @@ settings caused surprising deficiencies in some cases; and, in others default template is set, ``warewulfd`` looks for a template named "default". - If no init is specified, ``wwinit`` now looks for ``/sbin/init``, ``/etc/init``, and ``/bin/init``. -- `wwctl upgrade nodes --replace-overlays` now avoids adding the same overlay +- ``wwctl upgrade nodes --replace-overlays`` now avoids adding the same overlay multiple times to the same node or profile. - If, during ``wwctl overlay build``, a node has an empty system or runtime overlay, a warning is printed. diff --git a/userdocs/release/v4.6.2.rst b/userdocs/release/v4.6.2.rst index 1bb540e9..2f9baff6 100644 --- a/userdocs/release/v4.6.2.rst +++ b/userdocs/release/v4.6.2.rst @@ -3,7 +3,7 @@ v4.6.2 Release Notes ==================== v4.6.2 is a regularly-scheduled minor release in the v4.6.x series. It also -includes a preview implementation of the ability to provision the node image to +includes a preview implementation of the ability to provision the OS image to a local disk. Significant changes are described below. Additional changes are listed in the @@ -23,10 +23,10 @@ Disks may be provisioned using ``ignition``, as before; but may now also be provisioned using new ``sfdisk``, ``mkfs``, and ``mkswap`` overlays, and to mount configured file systems using ``systemd.mount`` and ``systemd.swap`` overlays. These overlays may also each be configured using overlay-specific -"resources", which may additional control compared to Warewulf's built-in disk, +"resources", which may provide additional control compared to Warewulf's built-in disk, partition, and file system configuration. -To support more complext partition layouts, ``wwctl set +To support more complex partition layouts, ``wwctl set --parttype`` has been added to specify non-default partition types. This is particularly useful when creating an EFI partition, which may be used in a future release to configure serverless local booting. diff --git a/userdocs/release/v4.6.3.rst b/userdocs/release/v4.6.3.rst index 07c4937c..09a45765 100644 --- a/userdocs/release/v4.6.3.rst +++ b/userdocs/release/v4.6.3.rst @@ -35,7 +35,7 @@ Overlay Autobuild The Warewulf server (when "autobuild overlays" is enabled) now automatically rebuilds overlays after node discovery, which resolves an issue where overlays were out-of-sync with the newly-discovered configuration. This is one step in a -more general effort to imrpove overlay autobuild so that manual overlay builds +more general effort to improve overlay autobuild so that manual overlay builds are less often required. Warewulf server configuration @@ -50,7 +50,7 @@ Ubuntu and Debian Support The ``netplan`` overlay is now much more feature-complete, with relative parity to the other network configuration overlays. This is one step in a more general -effort to support Wareewulf in Ubuntu and Debian environments. +effort to support Warewulf in Ubuntu and Debian environments. Other fixes =========== diff --git a/userdocs/release/v4.6.4.rst b/userdocs/release/v4.6.4.rst index 0bbabe79..ae9b5029 100644 --- a/userdocs/release/v4.6.4.rst +++ b/userdocs/release/v4.6.4.rst @@ -61,7 +61,7 @@ Network configuration fixes non-ethernet interfaces. - The NetworkManager overlay now sets ``ipv4:method=disabled`` if no address or route is specified. -- The ifcfg overlay now sets ``SLAVE=yes`` when `MASTER` is set. #1980 +- The ifcfg overlay now sets ``SLAVE=yes`` when ``MASTER`` is set. #1980 Other fixes =========== diff --git a/userdocs/release/v4.6.5.rst b/userdocs/release/v4.6.5.rst index 797dcbd2..af92e8b4 100644 --- a/userdocs/release/v4.6.5.rst +++ b/userdocs/release/v4.6.5.rst @@ -16,7 +16,7 @@ A new ``wwctl overlay info`` command lists variables used by an overlay template, including the ``wwctl set`` argument used to set that variable. -.. code-block: console +.. code-block:: console # wwctl overlay info ifcfg /etc/sysconfig/network-scripts/ifcfg.ww VARIABLE OPTION TYPE HELP @@ -40,12 +40,12 @@ Other variables and the overlay itself can also be documented explicitly within .. code-block:: text {{/* wwdoc: Configures chronyd to synchronize time with a specific time server. */}} - {{/* .Tags.ntp_server: the NTP server to synchonize with */}} + {{/* .Tags.ntp_server: the NTP server to synchronize with */}} wwctl image import --update =========================== -The ``wwctl image import`` come has included an ``--update`` option since +The ``wwctl image import`` command has included an ``--update`` option since v4.6.0, but it has not been functional. The option now imports an image on-top-of an existing image and is meant to be used to update an existing image with new files from an updated upstream or external image (e.g., by importing a @@ -88,21 +88,21 @@ development environment with libvirt. warewulf-dracut =============== -A bug between ``warewulf-dracut`` and the ``ignition`` overlay previously caused +A bug between ``warewulf-dracut`` and the ``ignition`` overlay previously prevented non-root filesystems from mounting when provisioning the image to disk. A fix in ``warewulf-dracut`` prevents ignition from running twice, allowing all file systems to mount properly. The ``wwinit`` dracut module also no longer runs the -wwinit module if root is not set to `root=wwclient*`. +wwinit module if root is not set to ``root=wwclient*``. These fixes require installing the updated ``warewulf-dracut`` package in the -node image. +OS image. Other fixes =========== -- Enhanced `wwctl configure tftp` to manage the SELinux context of TFTP directory. #1997 -- Enhanced overlay templates to support absolute paths with `{{ file }}`. #2055 +- Enhanced ``wwctl configure tftp`` to manage the SELinux context of TFTP directory. #1997 +- Enhanced overlay templates to support absolute paths with ``{{ file }}``. #2055 - Fix ImageDelete API not returning error when checking if image is used by nodes/profiles. #1705 -- Fix filesystem overwrite/force behavior in `mkfs` overlay. #2028 -- Write `$tftpdir/warewulf/grub.cfg` to `tftproot` as configured in `warewulf.conf`. #2055 -- Automatically create a GPT label when `sfdisk` overlay wipes disks. #2025 +- Fix filesystem overwrite/force behavior in ``mkfs`` overlay. #2028 +- Write ``$tftpdir/warewulf/grub.cfg`` to ``tftproot`` as configured in ``warewulf.conf``. #2055 +- Automatically create a GPT label when ``sfdisk`` overlay wipes disks. #2025 diff --git a/userdocs/server/api.rst b/userdocs/server/api.rst index 4b9053b4..fdc618e9 100644 --- a/userdocs/server/api.rst +++ b/userdocs/server/api.rst @@ -27,7 +27,7 @@ and passwords able to authenticate to the API. - name: admin password hash: $2b$05$5QVWDpiWE7L4SDL9CYdi3O/l6HnbNOLoXgY2sa1bQQ7aSBKdSqvsC -Passwords are stored as bcrypt2 hashses, which can be generated with ``mkpasswd``. +Passwords are stored as bcrypt2 hashes, which can be generated with ``mkpasswd``. .. code-block:: console diff --git a/userdocs/server/bootloaders.rst b/userdocs/server/bootloaders.rst index 8409ac82..214aa497 100644 --- a/userdocs/server/bootloaders.rst +++ b/userdocs/server/bootloaders.rst @@ -32,11 +32,11 @@ the Warewulf configuration process to configure the ``ipxe`` service. bios->iPXE [lhead=cluster0,label="iPXE.efi"]; - kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from node image"]; + kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from OS image"]; ipxe_cfg->kernel[ltail=cluster0,label="http"]; } -Starting in v4.5.0, Warewulf no longer includes an iPXE binary. In stead, by +Starting in v4.5.0, Warewulf no longer includes an iPXE binary. Instead, by default Warewulf uses the iPXE that comes with the host OS. Unfortunately, we’ve encountered a few instances where bugs in the OS-provided @@ -235,7 +235,7 @@ Warewulf as a technology preview. grubcfg[shape=record label="{grub.cfg|static under TFTP root}"]; grub->grubcfg[label="TFTP"]; } - kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from node image"]; + kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from OS image"]; grubcfg->kernel[ltail=cluster1,label="http"]; } @@ -245,7 +245,7 @@ advantage that secure boot can be used. That means that only the signed kernel of a distribution can be booted. This can be a huge security benefit for some scenarios. -In order to enable the grub boot method it has to be enabled in `warewulf.conf`. +In order to enable the grub boot method it has to be enabled in ``warewulf.conf``. .. code-block:: yaml @@ -276,14 +276,14 @@ Secure boot If secure boot is enabled at every step a signature is checked and the boot process fails if this check fails. The shim typically only includes the key for a single operating system, which means that each distribution needs separate -`shim` and `grub` executables. Warewulf extracts these binaries from the images. +``shim`` and ``grub`` executables. Warewulf extracts these binaries from the images. If the node is unknown to Warewulf or can't be identified during the TFTP boot phase, the shim/grub binaries of the host in which Warewulf is running are used. Install shim and efi -------------------- -`shim.efi` and `grub.efi` must be installed in the image for it to be +``shim.efi`` and ``grub.efi`` must be installed in the image for it to be booted by GRUB. .. code-block:: console @@ -308,15 +308,15 @@ diagram is the following: digraph G{ node [shape=box]; efi [shape=record label="{EFI|boots from URI defined in filename}"]; - shim [shape=record label="{shim.efi|replaces shim.efi with grubx64.efi in URI|extracted from node image}"]; - grub [shape=record label="{grub.efi|checks for grub.cfg|extracted from node image}"] - kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from node image"]; + shim [shape=record label="{shim.efi|replaces shim.efi with grubx64.efi in URI|extracted from OS image}"]; + grub [shape=record label="{grub.efi|checks for grub.cfg|extracted from OS image}"] + kernel [shape=record label="{kernel|ramdisk (root fs)|wwinit overlay}|extracted from OS image"]; efi->shim [label="http"]; shim->grub [label="http"]; grub->kernel [label="http"]; } -Warewulf delivers the initial `shim.efi` and `grub.efi` via http as taken +Warewulf delivers the initial ``shim.efi`` and ``grub.efi`` via http as taken directly from the node's assigned image. .. _booting with dracut: @@ -325,7 +325,7 @@ Two-stage boot: dracut ====================== Some systems, typically due to limitations in their BIOS or EFI firmware, are -unable to load image of a certain size directly with a traditional bootloader, +unable to load an image of a certain size directly with a traditional bootloader, either iPXE or GRUB. As a workaround for such systems, Warewulf can be configured to load a dracut initramfs from the image and to use that initramfs to load the full image. @@ -394,5 +394,5 @@ during boot. The wwinit module provisions to tmpfs. By default, tmpfs is permitted to use up to 50% of physical memory. This size limit may be adjusted using the kernel -argument `wwinit.tmpfs.size`. (This parameter is passed to the `size` option +argument ``wwinit.tmpfs.size``. (This parameter is passed to the ``size`` option during tmpfs mount. See ``tmpfs(5)`` for more details.) diff --git a/userdocs/server/configuration.rst b/userdocs/server/configuration.rst index cf602adf..08d88e5b 100644 --- a/userdocs/server/configuration.rst +++ b/userdocs/server/configuration.rst @@ -4,7 +4,7 @@ Server Configuration By default, the Warewulf server configuration is located at ``/etc/warewulf/warewulf.conf``. This is a YAML-formatted configuration file -used by to configured the Warewulf server itself and its external services. +used to configure the Warewulf server itself and its external services. An initial ``warewulf.conf`` is packaged with Warewulf. Each section is covered in detail below. @@ -109,7 +109,7 @@ warewulf can be specified at ``wwclient:port``.) Changing this option requires rebuilding node overlays and rebooting compute - nodes to configure them to use a privileged port for `wwclient`. + nodes to configure them to use a privileged port for ``wwclient``. * ``warewulf:update interval``: This defines the frequency (in seconds) with which the Warewulf client on the compute node fetches overlay updates. @@ -124,7 +124,7 @@ warewulf applied to the Warewulf server during configuration. (The host overlay is used to configure external services.) -* ``warewulf::grubboot``: Controls whether iPXE (default) or GRUB is used as the +* ``warewulf:grubboot``: Controls whether iPXE (default) or GRUB is used as the network bootloader. dhcp @@ -232,7 +232,7 @@ ssh *New in Warewulf v4.5.1* -SSH key types to generate during ``wwctl configure ssh``. This create the +SSH key types to generate during ``wwctl configure ssh``. This creates the appropriate host keys (stored in ``/etc/warewulf/keys/``) and authentication keys for passwordless ``ssh`` to cluster nodes. It also installs shell profiles ``/etc/profile.d/ssh_setup.csh`` and ``/etc/profile.d/ssh_setup.sh`` to @@ -248,7 +248,7 @@ Warewulf server. - rsa - dsa -* ``ssh:key types``: Warewulf generate host keys for each listed key type. +* ``ssh:key types``: Warewulf generates host keys for each listed key type. The first listed key type is used to generate authentication ssh keys. @@ -270,10 +270,10 @@ to operate in the host environment prior to deployment. * ``image mounts:dest``: The path in the image to use for the mount. -* ``image mounts::readonly``: Whether the mount should be read-only (``true``) +* ``image mounts:readonly``: Whether the mount should be read-only (``true``) or allow writes into the server path (``false``). -* ``image mounts::copy``: When ``true``, copy files into the image rather than +* ``image mounts:copy``: When ``true``, copy files into the image rather than mount. This is useful for initializing files with a starting value from the Warewulf server that should then be maintained as part of the image. @@ -299,8 +299,9 @@ Override paths to images, overlays, and other Warewulf components. * ``paths:sysconfdir``: The parent directory for the ``warewulf`` configuration directory, which stores ``warewulf.conf`` and ``nodes.conf``. -* ``paths::cachedir``: The parent directory for the ``warewulf`` cache of OCI - images during ``wwctl image import``. +* ``paths:cachedir``: The parent directory for the ``warewulf`` cache of OCI + images during ``wwctl image import``. The cache is stored at + ``$cachedir/warewulf`` and can be cleared with ``wwctl clean``. * ``paths:ipxesource``: Where to get iPXE binaries. These files are copied to ``warewulf.conf:tftp:tftproot`` by ``wwctl configure tftp``. @@ -359,7 +360,7 @@ hostfile There are no explicit "hostfile" configuration options in ``warewulf.conf``; but ``wwctl configure hostfile`` updates the Warewulf server's ``/etc/hosts`` file to include expected configuration for the server itself as well as the known -names of the cluster nodes and thier interfaces. +names of the cluster nodes and their interfaces. Entries from the Warewulf server's ``/etc/hosts`` file are distributed to cluster nodes by the "hosts" overlay. diff --git a/userdocs/server/dnsmasq.rst b/userdocs/server/dnsmasq.rst index b182ee8d..01a98458 100644 --- a/userdocs/server/dnsmasq.rst +++ b/userdocs/server/dnsmasq.rst @@ -5,7 +5,7 @@ Using dnsmasq As an experimental feature, it is possible to use ``dnsmasq`` instead of the ISC ``dhcpd`` server and ``TFTP`` server. -In order to keep the file ``/etc/dnsmasq.d/ww4-hosts.conf`` is created and must +In order to keep the configuration manageable, the file ``/etc/dnsmasq.d/ww4-hosts.conf`` is created and must be included in the main ``dnsmasq.conf`` via the ``conf-dir=/etc/dnsmasq.d`` option. diff --git a/userdocs/server/installation.rst b/userdocs/server/installation.rst index d4bb4d37..ea4c36ef 100644 --- a/userdocs/server/installation.rst +++ b/userdocs/server/installation.rst @@ -32,7 +32,7 @@ Container images Warewulf can be built in a Linux container. This can be especially useful for testing and development, or to replace traditional package installation. It is -also possible to only use the container for building and the install it in the +also possible to only use the container for building and then install it in the host system afterwards. For that look at the INSTALL, UNINSTALL and PURGE labels inside the `Dockerfile`_ diff --git a/userdocs/server/routes.rst b/userdocs/server/routes.rst index 9d75ec0f..24dbd98e 100644 --- a/userdocs/server/routes.rst +++ b/userdocs/server/routes.rst @@ -93,7 +93,7 @@ kernel is taken from the node's assigned image. ``/image/{wwid}`` ----------------- -Serves the raw node image file for the node identified by ``{wwid}``. +Serves the raw OS image file for the node identified by ``{wwid}``. **Query parameters:** ``assetkey``, ``uuid``, ``compress`` @@ -181,6 +181,12 @@ of the client's IP address against the kernel's ARP cache (``/proc/net/arp``). This route is intended for EFI HTTP Boot clients, where the firmware fetches a boot URI from DHCP and cannot perform variable substitution. +.. note:: + + On large clusters the kernel's default ARP cache limits may be exceeded, + causing node identification to fail. See + :ref:`arp-cache-overflow-on-large-clusters` for tuning guidance. + The ``{file}`` component determines what is served: * ``shim.efi``: Serves the ``shim.efi`` binary extracted from the node's diff --git a/userdocs/server/security.rst b/userdocs/server/security.rst index a0aaa068..5693e24b 100644 --- a/userdocs/server/security.rst +++ b/userdocs/server/security.rst @@ -11,7 +11,7 @@ Provisioning Security Provisioning is, by default, a relatively "insecure" process: there is generally nothing preventing a user on a cluster node from spoofing a provision request -and downloading the node image and overlays for inspection. If any of these +and downloading the OS image and overlays for inspection. If any of these include secrets (e.g., private keys) they are at risk of exposure. There are multiple ways to secure the Warewulf provisioning process: @@ -36,7 +36,7 @@ There are multiple ways to secure the Warewulf provisioning process: privileged (< 1024) TCP port. This prevents unprivileged cluster users from being able to retrieve the runtime overlay. -* When the nodes are booted via `shim` and `grub` Secure Boot can be enabled. +* When the nodes are booted via ``shim`` and ``grub`` Secure Boot can be enabled. This means that the nodes only boot the kernel which is provided by the distributor and also custom complied modules can't be loaded. @@ -50,7 +50,7 @@ SELinux The Warewulf server can be run with SELinux enabled in "targeted" and "enforcing" mode. -For more information about running SELinux-enabled cluster node images, see +For more information about running SELinux-enabled cluster OS images, see :ref:`SELinux-Enabled Images `. firewalld diff --git a/userdocs/server/wwctl.rst b/userdocs/server/wwctl.rst index 53f0cc10..ad6d9cfe 100644 --- a/userdocs/server/wwctl.rst +++ b/userdocs/server/wwctl.rst @@ -8,10 +8,12 @@ functionality. * ``configure``: configures the Warewulf server and its external services * ``node``: manages nodes in the cluster -* ``profiles``: defines common sets of node configuration which can be applied +* ``profile``: defines common sets of node configuration which can be applied to multiple nodes * ``image``: configures (node) images -* ``overlays``: manages overlays +* ``overlay``: manages overlays +* ``clean``: removes the OCI image cache and leftover overlay images from + deleted nodes ``wwctl`` also provides additional helpers for interacting with cluster nodes over SSH and IPMI. @@ -25,7 +27,7 @@ built-in help with either ``wwctl help`` or ``--help``. Hostlists ========= -Many of the commands (e.g., ``wwctl node list`` support a "hostlist" +Many of the commands (e.g., ``wwctl node list``) support a "hostlist" syntax for referring to multiple nodes at once. Hostlist expressions support both ranges and comma-separated numerical lists. @@ -47,12 +49,43 @@ through the following command : ================================================================================ n1 RUNTIME_OVERLAY __RUNTIME__.img.gz 16 -For each node, there are 4 different stages: +For each node, there are 7 different stages: +* **EFI** * **IPXE** * **KERNEL** +* **IMAGE** +* **INITRAMFS** * **SYSTEM_OVERLAY** * **RUNTIME_OVERLAY** You can use the ``wwctl node status`` to check communication between the Warewulf server (``warewulfd``) and the Warewulf client (``wwclient``). + +.. note:: A provisioning workflow might not use every stage. + +Maintenance +=========== + +``wwctl clean`` reclaims disk space by removing two categories of data that +accumulate over time: + +* The OCI blob cache, stored at ``$cachedir/warewulf`` (default: + ``/var/cache/warewulf``). This cache is populated during ``wwctl image + import`` and speeds up subsequent imports of images that share layers. It + is safe to remove: the next ``wwctl image import`` will re-download any + needed layers. + +* Provisioned overlay images for nodes that have been deleted from the node + database. These are stored under ``$wwprovisiondir/overlays`` and are no + longer needed once the corresponding node is removed. + +.. code-block:: console + + # wwctl clean + +``wwctl clean`` is particularly useful after deleting a large number of nodes, +or when disk space is limited on the Warewulf server. Note that ``wwctl clean`` +only removes the current cache location (``$cachedir/warewulf``); if you are +upgrading from v4.5.x, the legacy cache at ``$datastore/oci`` must be removed +manually (see :ref:`oci-blob-cache`). diff --git a/userdocs/troubleshooting/known-issues.rst b/userdocs/troubleshooting/known-issues.rst index bdc13e5d..9f796513 100644 --- a/userdocs/troubleshooting/known-issues.rst +++ b/userdocs/troubleshooting/known-issues.rst @@ -63,7 +63,7 @@ v4.6.0. `_ Image Size Considerations ========================= -Node images can grow quickly as packages and other files are added to them. Even +OS images can grow quickly as packages and other files are added to them. Even these larger images are often not an issue in modern environments; but some architectural limits exist that can impede the use of images larger than a few gigabytes. Workarounds exist for these issues in most circumstances: @@ -93,9 +93,59 @@ gigabytes. Workarounds exist for these issues in most circumstances: decompress the image rather than the kernel. * Some BIOS / firmware retain a "memory hole" feature for legacy devices, e.g., - reserving a 1MB block of memory at the 15MB-16MB address range. this feature - can interfere with booting stateless node images. + reserving a 1MB block of memory at the 15MB-16MB address range. This feature + can interfere with booting stateless OS images. If you are still getting "Not enough memory" or "No space left on device" errors, try disabling any "memory hole" features or updating your system BIOS or firmware. + +.. _arp-cache-overflow-on-large-clusters: + +ARP Cache Overflow on Large Clusters +===================================== + +On clusters with many nodes, the Linux kernel's default ARP cache limits may be +too low, causing the Warewulf server to log: + +.. code-block:: text + + neighbour: arp_cache: neighbor table overflow! + +The kernel manages the ARP cache with three garbage-collection thresholds: + +* ``net.ipv4.neigh.default.gc_thresh1`` -- no garbage collection below this + count (default: 128) +* ``net.ipv4.neigh.default.gc_thresh2`` -- garbage collection is triggered above + this count (default: 512) +* ``net.ipv4.neigh.default.gc_thresh3`` -- hard limit; entries are dropped above + this count (default: 1024) + +This is particularly relevant for Warewulf because ``warewulfd`` identifies nodes +by looking up their IP address in the kernel's ARP cache (``/proc/net/arp``) during +provisioning. If cache entries for cluster nodes are evicted, node identification +can fail. + +Increase the thresholds on the Warewulf server. As a starting point: + +.. code-block:: shell + + sysctl -w net.ipv4.neigh.default.gc_thresh2=1024 + sysctl -w net.ipv4.neigh.default.gc_thresh3=2048 + +For larger clusters (hundreds of nodes or more), higher values may be needed: + +.. code-block:: shell + + sysctl -w net.ipv4.neigh.default.gc_thresh2=2048 + sysctl -w net.ipv4.neigh.default.gc_thresh3=4096 + +To make the change persistent across reboots, create a file in ``/etc/sysctl.d/``: + +.. code-block:: shell + + cat > /etc/sysctl.d/90-warewulf-arp.conf << 'EOF' + net.ipv4.neigh.default.gc_thresh2 = 2048 + net.ipv4.neigh.default.gc_thresh3 = 4096 + EOF + sysctl -p /etc/sysctl.d/90-warewulf-arp.conf diff --git a/userdocs/troubleshooting/troubleshooting.rst b/userdocs/troubleshooting/troubleshooting.rst index 5ffc5f6d..acfcdcc5 100644 --- a/userdocs/troubleshooting/troubleshooting.rst +++ b/userdocs/troubleshooting/troubleshooting.rst @@ -66,10 +66,10 @@ cluster node's MAC address in place of 00:00:00:00:00:00.) - The ``kernel`` command fetches a kernel for later booting. - The ``imgextract`` command fetches and decompresses the images that will make - up the booted noe image. In a typical environment this is used to load a + up the booted OS image. In a typical environment this is used to load a minimal "initial ramdisk" which, then, boots the rest of the system. Warewulf, by default, loads the entire image as an initial ramdisk, and also loads the - system and runtime overlays at this time time. + system and runtime overlays at this time. - The ``boot`` command tells iPXE to boot the system with the given kernel and ramdisks. @@ -91,7 +91,7 @@ enabled. To do so, substitute the ``boot`` command above. You may be more familiar with specifying ``init=`` on the kernel command line. ``rdinit`` indicates "ramdisk init." Since Warewulf, by default, boots - the node image as an initial ramdisk, we must use ``rdinit=`` here. + the OS image as an initial ramdisk, we must use ``rdinit=`` here. GRUB ==== @@ -127,7 +127,7 @@ the port number if you have changed it from the default of 9873.) a typical environment this is used to load a minimal "initial ramdisk" which, then, boots the rest of the system. Warewulf, by default, loads the entire image as an initial ramdisk, and also loads the system and runtime overlays at - this time time. + this time. - The ``boot`` command tells GRUB to boot the system with the previously-defined configuration. @@ -149,7 +149,7 @@ enabled. To do so, substitute the ``linux`` command above. You may be more familiar with specifying ``init=`` on the kernel command line. ``rdinit`` indicates "ramdisk init." Since Warewulf, by default, boots - the node image as an initial ramdisk, we must use ``rdinit=`` here. + the OS image as an initial ramdisk, we must use ``rdinit=`` here. Dracut ====== @@ -195,12 +195,85 @@ investigate: - Sometimes you need to add ``should_exist: "true"`` for the swap partition as well. +Overlay Shadowing +================= + +When Warewulf introduced the distinction between distribution overlays and site +overlays, existing installations that had modified any distribution overlays +were left with those modified files in the site overlay directory (typically +``/var/lib/warewulf/overlays/``). Because a site overlay takes complete +precedence over a distribution overlay with the same name — with no merging of +individual files — the entire distribution overlay is shadowed. Any new files +or updates added to the distribution overlay in a subsequent Warewulf upgrade +will be hidden as long as a site overlay of the same name exists. + +To check whether any distribution overlays are being shadowed by site overlays, +use ``wwctl overlay list``, which includes a ``SITE`` column: + +.. code-block:: + + wwctl overlay list + +Any overlay showing ``true`` in the ``SITE`` column that you did not +intentionally create locally may be unintentionally shadowing its distribution +counterpart. + +To see which files are present in a site overlay, use the ``--all`` flag: + +.. code-block:: + + wwctl overlay list --all + +To see the filesystem paths of the overlays directly, use the ``--path`` flag: + +.. code-block:: + + wwctl overlay list --path + +If you determine that a site overlay is unintentionally shadowing a +distribution overlay, you can restore the distribution overlay by deleting the +site overlay. Back up any intentional local modifications first, then delete +the site overlay: + +.. code-block:: + + wwctl overlay delete + +``wwctl overlay delete`` only ever deletes site overlays, so this command is +safe to run without risk of removing the underlying distribution overlay. After +deleting the site overlay, ``wwctl overlay list`` should show ``false`` in the +``SITE`` column for that overlay, confirming that the distribution overlay is +now active. + Running Containers on Cluster Nodes =================================== -Some container runtimes, notably Podman, require file system features that are -not available in ``initrootfs``. Cluster nodes using Podman (and some other -container runtimes) should be configured with ``--root=tmpfs``. +Container runtimes such as Podman require filesystem features — most notably +OverlayFS support for image storage and container layers — that are not +available with the default ``initramfs`` root filesystem. To run Podman or +similar runtimes on cluster nodes, configure the node or profile to use +``tmpfs`` as the root filesystem: + +.. code-block:: shell + + # Apply to all nodes via a profile + wwctl profile set default --root=tmpfs + + # Or apply to a specific node + wwctl node set --root=tmpfs + +After changing the root filesystem type, reboot the affected nodes to apply +the new configuration. + +.. note:: + + The OS image itself must have Podman (or the desired container runtime) + installed. See :ref:`images` for guidance on customizing OS images. + +For information on tuning tmpfs memory usage and NUMA interleaving behavior, +see :ref:`tmpfs-and-numa` below. + +.. _tmpfs-and-numa: tmpfs and NUMA ============== @@ -208,7 +281,7 @@ tmpfs and NUMA Warewulf can optionally mount the root filesystem as ``tmpfs`` instead of the default ``initramfs``. Warewulf will add ``mpol=interleave`` to the mount point which will distribute the memory across all NUMA nodes. This avoids the -hotspotting that occurs when the default initramfs stores large node images on a +hotspotting that occurs when the default initramfs stores large OS images on a single NUMA node. To enable this, set the rootfs type to tmpfs: .. code-block:: shell @@ -228,8 +301,52 @@ argument: By default this is set to 50% of physical RAM. Note that tmpfs is required for SELinux overlays since initramfs cannot preserve SELinux contexts. +Because the root is ``tmpfs``, the kernel can also swap cold image pages to a +local swap device, freeing RAM for running workloads. This does not apply to the +default ``initramfs`` root (single-stage boot), where pages are pinned in memory +and cannot be swapped. See :ref:`swap-and-image-memory` for a complete walkthrough. + .. note:: On some systems, it may also be necessary to include the ``noefi`` kernel argument. This works around specific EFI firmware bugs that can prevent proper memory release during the transition from ``initramfs`` to ``tmpfs``. + +.. _oci-blob-cache: + +OCI Blob Cache +============== + +Warewulf caches OCI image layers on disk to speed up repeated ``wwctl image +import`` operations. The cache can grow large when many images — or many +versions of the same image — are imported over time. + +**v4.6 and later** + +The cache is stored at ``$cachedir/warewulf`` (default: ``/var/cache/warewulf`` +on RPM-based distributions). It contains files and directories such as +``blobs/``, ``index.json``, and ``oci-layout``. + +Use ``wwctl clean`` to remove the cache: + +.. code-block:: console + + # wwctl clean + +The cache is rebuilt automatically on the next ``wwctl image import``. + +**v4.5.x and earlier (legacy cache location)** + +In v4.5.x and earlier releases, the OCI blob cache was stored at +``$datastore/oci`` (default: ``/usr/share/oci`` in the Rocky Linux RPM +packages). This location is not removed by ``wwctl clean``. + +If you are upgrading from v4.5.x and want to reclaim the space used by the +old cache, you can safely delete this directory manually: + +.. code-block:: console + + # rm -rf /usr/share/oci + +Adjust the path if your installation used a non-default ``datastore`` +setting in ``warewulf.conf``. diff --git a/warewulf.spec.in b/warewulf.spec.in index 855e1821..df59e810 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -253,7 +253,7 @@ Warewulf is a stateless and diskless provisioning system for large clusters of bare metal and/or virtual systems. This subpackage contains a dracut module that can be used to generate an -initramfs that can fetch and boot a Warewulf node image from a Warewulf server. +initramfs that can fetch and boot a Warewulf OS image from a Warewulf server. %files dracut %defattr(-, root, root)