From 21333482e60154857f0e699d1dd2aefddae0063e Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Mon, 10 Feb 2025 19:11:52 -0700 Subject: [PATCH] Omit duplicate passwd and group entries in syncuser overlay Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 10 +-- internal/pkg/overlay/funcmap.go | 31 +++++++++ internal/pkg/overlay/funcmap_test.go | 47 ++++++++++++++ internal/pkg/overlay/overlay.go | 1 + overlays/syncuser/internal/nodes.conf | 7 +- overlays/syncuser/internal/syncuser_test.go | 72 ++++++++++++--------- overlays/syncuser/rootfs/etc/group.ww | 8 ++- overlays/syncuser/rootfs/etc/passwd.ww | 11 ++-- userdocs/contents/overlays.rst | 4 ++ userdocs/contents/templating.rst | 25 ++++++- 10 files changed, 171 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef792621..fab2c42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Added missing hostlist support for `wwctl node` and `wwctl overlay build`. #1635 - Added support for comma-separated hostlist patterns. #1635 - Added default value for `warewulf.conf:dhcp.template`. #1725 +- Added `UniqueField` template function. #829 ### Changed - Hide internal `wwctl completion` and `wwctl genconfig` commands. #1716 - Make .ww suffix optional during `wwctl overlay show --render`. #649 +- DHCP template generates as much of the subnet and range definition as possible. #1469 +- Updated overlay flags to `wwctl [--runtime-overlays|--system-overlays]`. #1495 +- syncuser overlay reads host passwd and group database from sysconfdir. #1736 +- syncuser overlay skips duplicate users and groups in passwd and group databases. #829 ### Fixed @@ -31,11 +36,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Removed partial support for regex searches in node and profile lists. #1635 - Remove redundant `wwctl genconfig completions` command. #1716 -### Changed - -- DHCP template generates as much of the subnet and range definition as possible. #1469 -- Updated overlay flags to `wwctl [--runtime-overlays|--system-overlays]`. #1495 - ## v4.6.0rc2, 2025-02-07 ### Added diff --git a/internal/pkg/overlay/funcmap.go b/internal/pkg/overlay/funcmap.go index 164de7de..717eadee 100644 --- a/internal/pkg/overlay/funcmap.go +++ b/internal/pkg/overlay/funcmap.go @@ -130,3 +130,34 @@ func importSoftlink(lnk string) string { func softlink(target string) string { return fmt.Sprintf("{{ /* softlink \"%s\" */ }}", target) } + +// UniqueField returns a filtered version of a multi-line input string. input is +// expected to be a field-separated format with one record per line (terminated +// by `\n`). Order of lines is preserved, with the first matching line taking +// precedence. +// +// For example, parsing /etc/passwd filter /etc/passwd for unique user names: +// +// Lines without the index field (e.g., blank lines) are always included in the +// output. +// +// UniqueField(":", 0, passwdContent) +func UniqueField(sep string, index int, input string) string { + inputLines := strings.Split(input, "\n") + var outputLines []string + found := make(map[string]bool) + for _, line := range inputLines { + inputFields := strings.Split(line, sep) + if len(inputFields) > index { + field := inputFields[index] + if field != "" { + if found[field] { + continue + } + found[field] = true + } + } + outputLines = append(outputLines, line) + } + return strings.Join(outputLines, "\n") +} diff --git a/internal/pkg/overlay/funcmap_test.go b/internal/pkg/overlay/funcmap_test.go index a4fbbef7..fb108cf0 100644 --- a/internal/pkg/overlay/funcmap_test.go +++ b/internal/pkg/overlay/funcmap_test.go @@ -59,3 +59,50 @@ func Test_createIgnitionJson(t *testing.T) { node := nodeInfos[0] assert.JSONEq(t, expected_json, createIgnitionJson(&node)) } + +func Test_UniqueField(t *testing.T) { + tests := map[string]struct { + input string + sep string + field int + output string + }{ + "empty": { + input: ``, + sep: ":", + field: 0, + output: ``, + }, + + "unique input": { + input: ` +name1:aaaa +name2:bbbb +`, + sep: ":", + field: 0, + output: ` +name1:aaaa +name2:bbbb +`, + }, + + "duplicate field": { + input: ` +name1: aaaa +name1: bbbb +`, + sep: ":", + field: 0, + output: ` +name1: aaaa +`, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tt.output, UniqueField(tt.sep, tt.field, tt.input)) + }) + } +} diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index 3f3995f7..0bca2140 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -500,6 +500,7 @@ func RenderTemplateFile(fileName string, data TemplateStruct) ( backupFile = false return "" }, + "UniqueField": UniqueField, } // Merge sprig.FuncMap with our FuncMap diff --git a/overlays/syncuser/internal/nodes.conf b/overlays/syncuser/internal/nodes.conf index 132a2f4a..181c42aa 100644 --- a/overlays/syncuser/internal/nodes.conf +++ b/overlays/syncuser/internal/nodes.conf @@ -1,2 +1,7 @@ nodes: - node1: {} + node1: + image name: rockylinux-9 + node2: + image name: rockylinux-9 + tags: + PasswordlessRoot: true diff --git a/overlays/syncuser/internal/syncuser_test.go b/overlays/syncuser/internal/syncuser_test.go index e852cc03..ee12dc64 100644 --- a/overlays/syncuser/internal/syncuser_test.go +++ b/overlays/syncuser/internal/syncuser_test.go @@ -11,35 +11,61 @@ import ( ) func Test_syncuserOverlay(t *testing.T) { - t.Skip("syncuser is not yet isolated from the host") - env := testenv.New(t) defer env.RemoveAll() env.ImportFile("etc/warewulf/nodes.conf", "nodes.conf") - env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/passwd.ww", "../../../../../overlays/syncuser/rootfs/etc/passwd.ww") - env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/group.ww", "../../../../../overlays/syncuser/rootfs/etc/group.ww") - env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/passwd", `root:x:0:0:root:/root:/bin/bash`) - env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/group", `root:x:0:`) + env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/passwd.ww", "../rootfs/etc/passwd.ww") + env.ImportFile("var/lib/warewulf/overlays/syncuser/rootfs/etc/group.ww", "../rootfs/etc/group.ww") + env.WriteFile("etc/passwd", ` +root:x:0:0:root:/root:/bin/bash +user:x:1000:1000:user:/home/user:/bin/bash +`) + env.WriteFile("etc/group", ` +root:x:0: +user:x:1000: +`) + env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/passwd", ` +root:x:0:0:root:/root:/bin/bash +`) + env.WriteFile("var/lib/warewulf/chroots/rockylinux-9/rootfs/etc/group", ` +root:x:0: +`) - tests := []struct { - name string + tests := map[string]struct { args []string log string }{ - { - name: "syncuser:passwd.ww", + "syncuser:passwd.ww": { args: []string{"--render", "node1", "syncuser", "etc/passwd.ww"}, - log: syncuser_passwd, + log: `backupFile: true +writeFile: true +Filename: etc/passwd +root:x:0:0:root:/root:/bin/bash +user:x:1000:1000:user:/home/user:/bin/bash +`, }, - { - name: "syncuser:group.ww", + "syncuser:passwd.ww (passwordless root)": { + args: []string{"--render", "node2", "syncuser", "etc/passwd.ww"}, + log: `backupFile: true +writeFile: true +Filename: etc/passwd +root::0:0:root:/root:/bin/bash +user:x:1000:1000:user:/home/user:/bin/bash +`, + }, + "syncuser:group.ww": { args: []string{"--render", "node1", "syncuser", "etc/group.ww"}, - log: syncuser_group, + log: `backupFile: true +writeFile: true +Filename: etc/group +root:x:0: +user:x:1000: +`, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for name, tt := range tests { + t.Run(name, func(t *testing.T) { cmd := show.GetCommand() cmd.SetArgs(tt.args) stdout := bytes.NewBufferString("") @@ -56,17 +82,3 @@ func Test_syncuserOverlay(t *testing.T) { }) } } - -const syncuser_passwd string = `backupFile: true -writeFile: true -Filename: etc/passwd -# Uncomment the following line to enable passwordless root login -# root::0:0:root:/root:/bin/bash -root:x:0:0:root:/root:/bin/bash -` - -const syncuser_group string = `backupFile: true -writeFile: true -Filename: etc/group -root:x:0: -` diff --git a/overlays/syncuser/rootfs/etc/group.ww b/overlays/syncuser/rootfs/etc/group.ww index 007ca265..a2ee9bc8 100644 --- a/overlays/syncuser/rootfs/etc/group.ww +++ b/overlays/syncuser/rootfs/etc/group.ww @@ -1,2 +1,6 @@ -{{IncludeFrom $.ImageName "/etc/group"}} -{{Include "/etc/group"}} +{{ + printf "%s\n%s" + (IncludeFrom $.ImageName "/etc/group" | trim) + (Include (printf "%s/%s" .Paths.Sysconfdir "group") | trim) + | UniqueField ":" 0 | trim +}} diff --git a/overlays/syncuser/rootfs/etc/passwd.ww b/overlays/syncuser/rootfs/etc/passwd.ww index ec3c9964..7655373b 100644 --- a/overlays/syncuser/rootfs/etc/passwd.ww +++ b/overlays/syncuser/rootfs/etc/passwd.ww @@ -1,4 +1,7 @@ -# Uncomment the following line to enable passwordless root login -# root::0:0:root:/root:/bin/bash -{{IncludeFrom $.ImageName "/etc/passwd"}} -{{Include "/etc/passwd"}} +{{- $sources := list }} +{{- if and .Tags.PasswordlessRoot (eq (lower .Tags.PasswordlessRoot) "true") }} +{{- $sources = append $sources "root::0:0:root:/root:/bin/bash" }} +{{- end }} +{{- $sources = append $sources (IncludeFrom $.ImageName "/etc/passwd" | trim) }} +{{- $sources = append $sources (Include (printf "%s/%s" .Paths.Sysconfdir "passwd") | trim) }} +{{- join "\n" $sources | UniqueField ":" 0 | trim }} diff --git a/userdocs/contents/overlays.rst b/userdocs/contents/overlays.rst index edcc0cf9..5587fcba 100644 --- a/userdocs/contents/overlays.rst +++ b/userdocs/contents/overlays.rst @@ -161,6 +161,10 @@ all users on both the Warewulf server and from the image. To function properly, ``wwctl image syncuser`` must have also been run on the image image to synchronize its user and group IDs with those of the server. +If a ``PasswordlessRoot`` tag is set to "true", the overlay will also insert a +"passwordless" root entry. This can be particularly useful for accessing a +cluster node when its network interface is not properly configured. + ignition -------- diff --git a/userdocs/contents/templating.rst b/userdocs/contents/templating.rst index a5459a42..93c88716 100644 --- a/userdocs/contents/templating.rst +++ b/userdocs/contents/templating.rst @@ -135,13 +135,13 @@ The following template will create a file called {{ end -}} -Special Commands ----------------- +Special Functions +----------------- Include ^^^^^^^ -A file from the host can be include with following template +A file from the host can be included with following template .. code-block:: @@ -228,6 +228,25 @@ readlink Evaluates the soft link on the Warewulf server and returns the target. +UniqueField +^^^^^^^^^^^ + +UniqueField returns a filtered version of a multi-line input string. input is +expected to be a field-separated format with one record per line (terminated by +`\n`). Order of lines is preserved, with the first matching line taking +precedence. + +For example, the following template snippet has been used in the ``syncuser`` overlay +to generate a combined ``/etc/passwd``. + +.. code-block:: + + {{ + printf "%s\n%s" + (IncludeFrom $.ImageName "/etc/passwd" | trim) + (Include (printf "%s/%s" .Paths.Sysconfdir "passwd") | trim) + | UniqueField ":" 0 | trim + }} Node specific files -------------------