kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1736 from anderbubble/syncuser-improvements

Browse Source 
syncuser improvements
This commit is contained in:
Christian Goll
2025-02-14 08:47:13 +01:00
committed by GitHub
parent a686ddb7ad eb6ce7d86c
commit 428b06463f
22 changed files with 243 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
CHANGELOG.md
View File

@@ -11,11 +11,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Added missing hostlist support for `wwctl node` and `wwctl overlay build`. #1635
- Added support for comma-separated hostlist patterns. #1635
- Added default value for `warewulf.conf:dhcp.template`. #1725
- Added `UniqueField` template function. #829
- Added `wwctl image build --syncuser`. #1321
### Changed
- Hide internal `wwctl completion` and `wwctl genconfig` commands. #1716
- Make .ww suffix optional during `wwctl overlay show --render`. #649
- DHCP template generates as much of the subnet and range definition as possible. #1469
- Updated overlay flags to `wwctl <node|profile> <add|set> [--runtime-overlays|--system-overlays]`. #1495
- syncuser overlay reads host passwd and group database from sysconfdir. #1736
- syncuser overlay skips duplicate users and groups in passwd and group databases. #829
- `wwctl image syncuser --write` is true by default. #1736
- Update syncuser documentation. #1736
### Fixed
@@ -30,11 +38,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Removed partial support for regex searches in node and profile lists. #1635
- Remove redundant `wwctl genconfig completions` command. #1716
### Changed
- DHCP template generates as much of the subnet and range definition as possible. #1469
- Updated overlay flags to `wwctl <node|profile> <add|set> [--runtime-overlays|--system-overlays]`. #1495
- Remove syncuser warning messages in `wwctl` that assume its use. #1321
- Remove syncuser from the list of default runtime overlays. #1322
## v4.6.0rc2, 2025-02-07

1
etc/nodes.conf
View File

@@ -4,7 +4,6 @@ nodeprofiles:
runtime overlay:
- hosts
- ssh.authorized_keys
- syncuser
system overlay:
- wwinit
- wwclient

15
internal/app/wwctl/image/build/main.go
View File

@@ -1,16 +1,27 @@
package build
import (
"fmt"
"github.com/spf13/cobra"
"github.com/warewulf/warewulf/internal/pkg/api/image"
apiimage "github.com/warewulf/warewulf/internal/pkg/api/image"
"github.com/warewulf/warewulf/internal/pkg/api/routes/wwapiv1"
"github.com/warewulf/warewulf/internal/pkg/image"
)
func CobraRunE(cmd *cobra.Command, args []string) error {
if SyncUser {
for _, name := range args {
if err := image.Syncuser(name, true); err != nil {
return fmt.Errorf("syncuser error: %w", err)
}
}
}
cbp := &wwapiv1.ImageBuildParameter{
ImageNames: args,
Force: BuildForce,
All: BuildAll,
}
return image.ImageBuild(cbp)
return apiimage.ImageBuild(cbp)
}

2
internal/app/wwctl/image/build/root.go
View File

@@ -23,11 +23,13 @@ var (
}
BuildForce bool
BuildAll bool
SyncUser bool
)
func init() {
baseCmd.PersistentFlags().BoolVarP(&BuildAll, "all", "a", false, "(re)Build all images")
baseCmd.PersistentFlags().BoolVarP(&BuildForce, "force", "f", false, "Force rebuild, even if it isn't necessary")
baseCmd.PersistentFlags().BoolVar(&SyncUser, "syncuser", false, "Synchronize UIDs/GIDs from host to image")
}
// GetRootCommand returns the root cobra.Command for the application.

17
internal/app/wwctl/image/exec/main.go
View File

@@ -130,9 +130,6 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
afterPasswdTime := getTime(path.Join(imagePath, "/etc/passwd"))
wwlog.Debug("passwdTime: %v", afterPasswdTime)
if beforePasswdTime.Before(afterPasswdTime) {
if !SyncUser {
wwlog.Warn("/etc/passwd has been modified, maybe you want to run syncuser")
}
userdbChanged = true
}
}
@@ -140,16 +137,16 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
afterGroupTime := getTime(path.Join(imagePath, "/etc/group"))
wwlog.Debug("groupTime: %v", afterGroupTime)
if beforeGroupTime.Before(afterGroupTime) {
if !SyncUser {
wwlog.Warn("/etc/group has been modified, maybe you want to run syncuser")
}
userdbChanged = true
}
}
if userdbChanged && SyncUser {
err = image.SyncUids(imageName, false)
if err != nil {
wwlog.Error("Error in user sync, fix error and run 'syncuser' manually: %s", err)
if SyncUser {
if userdbChanged {
if err = image.Syncuser(imageName, false); err != nil {
wwlog.Error("syncuser error: %s", err)
}
} else {
wwlog.Info("Skipping syncuser (passwd and group files not changed)")
}
}

4
internal/app/wwctl/image/syncuser/main.go
View File

@@ -15,14 +15,14 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
if !image.ValidName(imageName) {
return fmt.Errorf("%s is not a valid image", imageName)
}
err := image.SyncUids(imageName, write)
err := image.Syncuser(imageName, write)
if err != nil {
return fmt.Errorf("error in synchronize: %s", err)
}
if write && !build {
// when write = true and build = false, we will print a warnning, this is the default case
wwlog.Warn("Syncuser is completed, please remember to rebuild image or add `--build` flag for automatic rebuild after syncuser")
wwlog.Warn("Syncuser is completed. Rebuild image or add `--build` flag for automatic rebuild after syncuser.")
} else if write && build {
// if write = true and build = true, then it'll trigger the image build after sync
cbp := &wwapiv1.ImageBuildParameter{

2
internal/app/wwctl/image/syncuser/root.go
View File

@@ -29,7 +29,7 @@ uid/gid collision is detected. File ownerships are also changed.`,
)
func init() {
baseCmd.PersistentFlags().BoolVar(&write, "write", false, "Synchronize uis/gids and write files in image")
baseCmd.PersistentFlags().BoolVar(&write, "write", true, "Synchronize uis/gids and write files in image")
baseCmd.PersistentFlags().BoolVar(&build, "build", false, "Build image after syncuser is completed")
}

4
internal/pkg/api/image/image.go
View File

@@ -193,9 +193,9 @@ func ImageImport(cip *wwapiv1.ImageImportParameter) (imageName string, err error
}
if cip.SyncUser {
err = image.SyncUids(cip.Name, true)
err = image.Syncuser(cip.Name, true)
if err != nil {
err = fmt.Errorf("error in user sync, fix error and run 'syncuser' manually: %s", err)
err = fmt.Errorf("syncuser error: %w", err)
return
}
}

25
internal/pkg/image/syncuids.go → internal/pkg/image/syncuser.go
View File

@@ -12,14 +12,12 @@ import (
"syscall"
"github.com/pkg/errors"
"github.com/warewulf/warewulf/internal/pkg/config"
"github.com/warewulf/warewulf/internal/pkg/util"
"github.com/warewulf/warewulf/internal/pkg/wwlog"
)
const passwdPath = "/etc/passwd"
const groupPath = "/etc/group"
// SyncUids updates the /etc/passwd and /etc/group files in the
// Syncuser updates the /etc/passwd and /etc/group files in the
// image identified by imageName by installing the equivalent
// files from the host and appending names only in the
// image. Files in the image are updated to match the new
@@ -33,14 +31,15 @@ const groupPath = "/etc/group"
// A conflict arises if the image has an entry with the same id as
// an entry in the host and the host does not have an entry with the
// same name. In this case, an error is returned.
func SyncUids(imageName string, write bool) error {
wwlog.Debug("SyncUids(imageName=%v, write=%v)", imageName, write)
func Syncuser(imageName string, write bool) error {
wwlog.Debug("Syncuser(imageName=%v, write=%v)", imageName, write)
conf := config.Get()
imagePath := RootFsDir(imageName)
imagePasswdPath := path.Join(imagePath, passwdPath)
imageGroupPath := path.Join(imagePath, groupPath)
imagePasswdPath := path.Join(imagePath, "/etc/passwd")
imageGroupPath := path.Join(imagePath, "/etc/group")
passwdSync := make(syncDB)
if err := passwdSync.readFromHost(passwdPath); err != nil {
if err := passwdSync.readFromHost(path.Join(conf.Paths.Sysconfdir, "passwd")); err != nil {
return err
}
if err := passwdSync.readFromimage(imagePasswdPath); err != nil {
@@ -51,7 +50,7 @@ func SyncUids(imageName string, write bool) error {
}
groupSync := make(syncDB)
if err := groupSync.readFromHost(groupPath); err != nil {
if err := groupSync.readFromHost(path.Join(conf.Paths.Sysconfdir, "group")); err != nil {
return err
}
if err := groupSync.readFromimage(imageGroupPath); err != nil {
@@ -78,16 +77,16 @@ func SyncUids(imageName string, write bool) error {
if err := groupSync.chownGroupFiles(); err != nil {
return err
}
if err := passwdSync.update(imagePasswdPath, passwdPath); err != nil {
if err := passwdSync.update(imagePasswdPath, "/etc/passwd"); err != nil {
return err
}
if err := groupSync.update(imageGroupPath, groupPath); err != nil {
if err := groupSync.update(imageGroupPath, "/etc/group"); err != nil {
return err
}
wwlog.Info("uid/gid synced for image %s", imageName)
} else {
if passwdSync.needsSync() || groupSync.needsSync() {
wwlog.Info("uid/gid not synced: run `wwctl image syncuser --write %s`", imageName)
wwlog.Info("uid/gid not synced: run `wwctl image syncuser --write=true %s` to synchronize", imageName)
} else {
wwlog.Info("uid/gid already synced")
}

31
internal/pkg/overlay/funcmap.go
View File

@@ -130,3 +130,34 @@ func importSoftlink(lnk string) string {
func softlink(target string) string {
return fmt.Sprintf("{{ /* softlink \"%s\" */ }}", target)
}
// UniqueField returns a filtered version of a multi-line input string. input is
// expected to be a field-separated format with one record per line (terminated
// by `\n`). Order of lines is preserved, with the first matching line taking
// precedence.
//
// For example, parsing /etc/passwd filter /etc/passwd for unique user names:
//
// Lines without the index field (e.g., blank lines) are always included in the
// output.
//
// UniqueField(":", 0, passwdContent)
func UniqueField(sep string, index int, input string) string {
inputLines := strings.Split(input, "\n")
var outputLines []string
found := make(map[string]bool)
for _, line := range inputLines {
inputFields := strings.Split(line, sep)
if len(inputFields) > index {
field := inputFields[index]
if field != "" {
if found[field] {
continue
}
found[field] = true
}
}
outputLines = append(outputLines, line)
}
return strings.Join(outputLines, "\n")
}

47
internal/pkg/overlay/funcmap_test.go
View File

@@ -59,3 +59,50 @@ func Test_createIgnitionJson(t *testing.T) {
node := nodeInfos[0]
assert.JSONEq(t, expected_json, createIgnitionJson(&node))
}
func Test_UniqueField(t *testing.T) {
tests := map[string]struct {
input string
sep string
field int
output string
}{
"empty": {
input: ``,
sep: ":",
field: 0,
output: ``,
},
"unique input": {
input: `
name1:aaaa
name2:bbbb
`,
sep: ":",
field: 0,
output: `
name1:aaaa
name2:bbbb
`,
},
"duplicate field": {
input: `
name1: aaaa
name1: bbbb
`,
sep: ":",
field: 0,
output: `
name1: aaaa
`,
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
assert.Equal(t, tt.output, UniqueField(tt.sep, tt.field, tt.input))
})
}
}

1
internal/pkg/overlay/overlay.go
View File

@@ -500,6 +500,7 @@ func RenderTemplateFile(fileName string, data TemplateStruct) (
backupFile = false
return ""
},
"UniqueField": UniqueField,
}
// Merge sprig.FuncMap with our FuncMap

7
overlays/syncuser/internal/nodes.conf
View File

@@ -1,2 +1,7 @@
nodes:
node1: {}
node1:
image name: rockylinux-9
node2:
image name: rockylinux-9
tags:
PasswordlessRoot: true

72
overlays/syncuser/internal/syncuser_test.go
View File

@@ -11,35 +11,61 @@ import (
)
func Test_syncuserOverlay(t *testing.T) {
t.Skip("syncuser is not yet isolated from the host")
env := testenv.New(t)
defer env.RemoveAll()
env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf")
env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/passwd.ww", "../../../../../overlays/syncuser/rootfs/etc/passwd.ww")
env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/group.ww", "../../../../../overlays/syncuser/rootfs/etc/group.ww")
env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/passwd", `root:x:0:0:root:/root:/bin/bash`)
env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/group", `root:x:0:`)
env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/passwd.ww", "../rootfs/etc/passwd.ww")
env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/group.ww", "../rootfs/etc/group.ww")
env.WriteFile("etc/passwd", `
root:x:0:0:root:/root:/bin/bash
user:x:1000:1000:user:/home/user:/bin/bash
`)
env.WriteFile("etc/group", `
root:x:0:
user:x:1000:
`)
env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/passwd", `
root:x:0:0:root:/root:/bin/bash
`)
env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/group", `
root:x:0:
`)
tests := []struct {
name string
tests := map[string]struct {
args []string
log string
}{
{
name: "syncuser:passwd.ww",
"syncuser:passwd.ww": {
args: []string{"--render", "node1", "syncuser", "etc/passwd.ww"},
log: syncuser_passwd,
log: `backupFile: true
writeFile: true
Filename: etc/passwd
root:x:0:0:root:/root:/bin/bash
user:x:1000:1000:user:/home/user:/bin/bash
`,
},
{
name: "syncuser:group.ww",
"syncuser:passwd.ww (passwordless root)": {
args: []string{"--render", "node2", "syncuser", "etc/passwd.ww"},
log: `backupFile: true
writeFile: true
Filename: etc/passwd
root::0:0:root:/root:/bin/bash
user:x:1000:1000:user:/home/user:/bin/bash
`,
},
"syncuser:group.ww": {
args: []string{"--render", "node1", "syncuser", "etc/group.ww"},
log: syncuser_group,
log: `backupFile: true
writeFile: true
Filename: etc/group
root:x:0:
user:x:1000:
`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
cmd := show.GetCommand()
cmd.SetArgs(tt.args)
stdout := bytes.NewBufferString("")
@@ -56,17 +82,3 @@ func Test_syncuserOverlay(t *testing.T) {
})
}
}
const syncuser_passwd string = `backupFile: true
writeFile: true
Filename: etc/passwd
# Uncomment the following line to enable passwordless root login
# root::0:0:root:/root:/bin/bash
root:x:0:0:root:/root:/bin/bash
`
const syncuser_group string = `backupFile: true
writeFile: true
Filename: etc/group
root:x:0:
`

8
overlays/syncuser/rootfs/etc/group.ww
View File

@@ -1,2 +1,6 @@
{{IncludeFrom $.ImageName "/etc/group"}}
{{Include "/etc/group"}}
{{
printf "%s\n%s"
(IncludeFrom $.ImageName "/etc/group" | trim)
(Include (printf "%s/%s" .Paths.Sysconfdir "group") | trim)
| UniqueField ":" 0 | trim
}}

11
overlays/syncuser/rootfs/etc/passwd.ww
View File

@@ -1,4 +1,7 @@
# Uncomment the following line to enable passwordless root login
# root::0:0:root:/root:/bin/bash
{{IncludeFrom $.ImageName "/etc/passwd"}}
{{Include "/etc/passwd"}}
{{- $sources := list }}
{{- if and .Tags.PasswordlessRoot (eq (lower .Tags.PasswordlessRoot) "true") }}
{{- $sources = append $sources "root::0:0:root:/root:/bin/bash" }}
{{- end }}
{{- $sources = append $sources (IncludeFrom $.ImageName "/etc/passwd" | trim) }}
{{- $sources = append $sources (Include (printf "%s/%s" .Paths.Sysconfdir "passwd") | trim) }}
{{- join "\n" $sources | UniqueField ":" 0 | trim }}

35
userdocs/contents/images.rst
View File

@@ -196,27 +196,30 @@ See ProxyFromEnvironment_ For more information.
Syncuser
========
At import time Warewulf checks if the names of the users on the host
match the users and UIDs/GIDs in the imported image. If there is
mismatch, the import command will print out a warning. By setting the
``--syncuser`` flag you advise Warewulf to try to syncronize the users
from the host to the image, which means that ``/etc/passwd`` and
``/etc/group`` of the imported image are updated and all the files
belonging to these UIDs and GIDs will also be updated.
Warewulf can optionally synchronize UIDs and GIDs from the Warewulf server to an
image. This can be particularly useful when there is no central directory (e.g.,
an LDAP server). Some system services (notably "munge") require a user to have
the same UID across all nodes.
A check if the users of the host and image matches can be
triggered with the ``syncuser`` command.
With the addition of the "syncuser" overlay, Warewulf syncuser also supports
defining local users on the Warewulf server for synchronization to cluster
nodes.
If there is mismatch between the server and the image, the import command will
print out a warning.
Syncuser may be invoked during image import, exec, shell, or build.
.. code-block:: console
# wwctl image syncuser image-name
# wwctl image import --syncuser docker://ghcr.io/warewulf/warewulf-rockylinux:9 rockylinux-9
# wwctl image exec --syncuser rockylinux-9 -- /usr/bin/echo "Hello, world!"
# wwctl image shell --syncuser rockylinux-9
# wwctl image build --syncuser rockylinux-9
# wwctl image syncuser rockylinux-9
With the ``--write`` flag it will update the image to match the
user database of the host as described above.
.. code-block:: console
wwctl image syncuser --write image-name
After syncuser, ``/etc/passwd`` and ``/etc/group`` in the image are updated, and
permissions on files belonging to these UIDs and GIDs are updated to match.
Listing All Imported Images
===========================

2
userdocs/contents/ipmi.rst
View File

@@ -103,7 +103,7 @@ Node View
n001 cluster -- --
n001 image default sle-micro-5.3
n001 ipxe -- (default)
n001 runtime -- (hosts,ssh.authorized_keys,syncuser)
n001 runtime -- (hosts,ssh.authorized_keys)
n001 wwinit -- (wwinit,wwclient,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,NetworkManager,debian.interfaces,wicked,ignition)
n001 root -- (initramfs)
n001 discoverable -- --

4
userdocs/contents/nodeconfig.rst
View File

@@ -87,7 +87,7 @@ You can also see the node's full attribute list by specifying the
n001 cluster -- --
n001 image default sle-micro-5.3
n001 ipxe -- (default)
n001 runtime -- (hosts,ssh.authorized_keys,syncuser)
n001 runtime -- (hosts,ssh.authorized_keys)
n001 wwinit -- (wwinit,wwclient,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,NetworkManager,debian.interfaces,wicked,ignition)
n001 root -- (initramfs)
n001 discoverable -- --
@@ -165,7 +165,7 @@ image and network:
n001 cluster -- --
n001 image default sle-micro-5.3
n001 ipxe -- (default)
n001 runtime -- (hosts,ssh.authorized_keys,syncuser)
n001 runtime -- (hosts,ssh.authorized_keys)
n001 wwinit -- (wwinit,wwclient,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,NetworkManager,debian.interfaces,wicked,ignition)
n001 root -- (initramfs)
n001 discoverable -- --

12
userdocs/contents/overlays.rst
View File

@@ -157,9 +157,15 @@ syncuser
--------
The **syncuser** overlay updates ``/etc/passwd`` and ``/etc/group`` to include
all users on both the Warewulf server and from the image. To function
properly, ``wwctl image syncuser`` must have also been run on the image
image to synchronize its user and group IDs with those of the server.
all users on both the Warewulf server and from the image.
To function properly, ``wwctl image syncuser`` (or the ``--syncuser`` option
during ``import``, ``exec``, ``shell``, or ``build``) must have also been run on
the image to synchronize its user and group IDs with those of the server.
If a ``PasswordlessRoot`` tag is set to "true", the overlay will also insert a
"passwordless" root entry. This can be particularly useful for accessing a
cluster node when its network interface is not properly configured.
ignition
--------

2
userdocs/contents/setup.rst
View File

@@ -118,7 +118,7 @@ You can check the result with ``wwctl node list``.
deliverynet Comment default This profile is automatically included for each node
deliverynet ImageName default leap15.5
deliverynet Ipxe -- (default)
deliverynet RuntimeOverlay -- (hosts,ssh.authorized_keys,syncuser)
deliverynet RuntimeOverlay -- (hosts,ssh.authorized_keys)
deliverynet SystemOverlay -- (wwinit,wwclient,fstab,hostname,ssh.host_keys,issue,resolv,udev.netname,systemd.netname,ifcfg,NetworkManager,debian.interfaces,wicked,ignition)
deliverynet Root -- (initramfs)
deliverynet Init -- (/sbin/init)

25
userdocs/contents/templating.rst
View File

@@ -135,13 +135,13 @@ The following template will create a file called
</interface>
{{ end -}}
Special Commands
----------------
Special Functions
-----------------
Include
^^^^^^^
A file from the host can be include with following template
A file from the host can be included with following template
.. code-block::
@@ -228,6 +228,25 @@ readlink
Evaluates the soft link on the Warewulf server and returns the target.
UniqueField
^^^^^^^^^^^
UniqueField returns a filtered version of a multi-line input string. input is
expected to be a field-separated format with one record per line (terminated by
`\n`). Order of lines is preserved, with the first matching line taking
precedence.
For example, the following template snippet has been used in the ``syncuser`` overlay
to generate a combined ``/etc/passwd``.
.. code-block::
{{
printf "%s\n%s"
(IncludeFrom $.ImageName "/etc/passwd" | trim)
(Include (printf "%s/%s" .Paths.Sysconfdir "passwd") | trim)
| UniqueField ":" 0 | trim
}}
Node specific files
-------------------