From 342236b9b92d7049b578cb0db900a4b3c6011435 Mon Sep 17 00:00:00 2001 From: Christian Goll Date: Wed, 20 Dec 2023 15:37:06 +0100 Subject: [PATCH] updated test for yaml/v3 Signed-off-by: Christian Goll --- internal/app/wwctl/node/add/main.go | 1 - internal/app/wwctl/node/set/main_test.go | 111 ++++++++++++- internal/app/wwctl/profile/add/main_test.go | 4 +- internal/pkg/api/apiconfig/client_server.go | 2 +- internal/pkg/api/node/set.go | 2 +- internal/pkg/api/profile/set.go | 11 +- internal/pkg/node/constructors.go | 56 +++---- internal/pkg/node/constuctors_test.go | 70 ++++++--- internal/pkg/node/datastructure.go | 5 +- internal/pkg/node/flags.go | 20 +-- internal/pkg/node/hash_test.go | 2 +- internal/pkg/node/list.go | 130 +++++++++++----- internal/pkg/node/methods.go | 163 ++++++++++---------- internal/pkg/node/modifiers.go | 4 +- internal/pkg/node/util_test.go | 4 +- internal/pkg/warewulfd/nodedb.go | 15 +- internal/pkg/warewulfd/provision_test.go | 37 ++--- internal/pkg/wwtype/wwbool.go | 100 +++++------- 18 files changed, 433 insertions(+), 304 deletions(-) diff --git a/internal/app/wwctl/node/add/main.go b/internal/app/wwctl/node/add/main.go index 55041457..f524b1cf 100644 --- a/internal/app/wwctl/node/add/main.go +++ b/internal/app/wwctl/node/add/main.go @@ -25,7 +25,6 @@ func CobraRunE(vars *variables) func(cmd *cobra.Command, args []string) error { if !node.ObjectIsEmpty(vars.nodeConf.NetDevs["UNDEF"]) { netDev := *vars.nodeConf.NetDevs["UNDEF"] vars.nodeConf.NetDevs[vars.netName] = &netDev - fmt.Println("not empty") } delete(vars.nodeConf.NetDevs, "UNDEF") if vars.fsName != "" { diff --git a/internal/app/wwctl/node/set/main_test.go b/internal/app/wwctl/node/set/main_test.go index 4a28f286..0c03c783 100644 --- a/internal/app/wwctl/node/set/main_test.go +++ b/internal/app/wwctl/node/set/main_test.go @@ -10,6 +10,7 @@ import ( "github.com/warewulf/warewulf/internal/pkg/node" "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/warewulfd" + "github.com/warewulf/warewulf/internal/pkg/wwlog" ) type test_description struct { @@ -24,7 +25,7 @@ type test_description struct { func run_test(t *testing.T, test test_description) { env := testenv.New(t) defer env.RemoveAll(t) - // wwlog.SetLogLevel(wwlog.DEBUG) + wwlog.SetLogLevel(wwlog.DEBUG) env.WriteFile(t, "etc/warewulf/nodes.conf", test.inDB) warewulfd.SetNoDaemon() name := test.name @@ -74,6 +75,7 @@ nodes: `} run_test(t, test) } + func Test_Node_Unset(t *testing.T) { test := test_description{ args: []string{"--comment=UNDEF", "n01"}, @@ -96,6 +98,113 @@ nodes: run_test(t, test) } +func Test_Set_Ipmi_Write_Explicit(t *testing.T) { + test := test_description{ + args: []string{"--ipmiwrite", "true", "n01"}, + wantErr: false, + stdout: "", + inDB: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: {} +`, + outDb: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: + ipmi: + write: "true" +`} + run_test(t, test) +} +func Test_Set_Ipmi_Write_Implicit(t *testing.T) { + test := test_description{ + args: []string{"--ipmiwrite", "n01"}, + wantErr: false, + stdout: "", + inDB: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: {} +`, + outDb: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: + ipmi: + write: "true" +`} + run_test(t, test) +} + +func Test_Unset_Ipmi_Write(t *testing.T) { + test := test_description{ + args: []string{"--ipmiwrite=UNDEF", "n01"}, + wantErr: false, + stdout: "", + inDB: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: + ipmi: + write: "true" +`, + outDb: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: {} +`} + run_test(t, test) +} +func Test_Unset_Ipmi_Write_False(t *testing.T) { + test := test_description{ + args: []string{"--ipmiwrite=UNDEF", "n01"}, + wantErr: false, + stdout: "", + inDB: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: + ipmi: + write: "false" +`, + outDb: `WW_INTERNAL: 43 +nodeprofiles: {} +nodes: + n01: {} +`} + run_test(t, test) +} +func Test_Ipmi_Hidden_False(t *testing.T) { + test := test_description{ + args: []string{"--ipmiwrite=false", "n01"}, + wantErr: false, + stdout: "", + inDB: `WW_INTERNAL: 43 +nodeprofiles: + default: + ipmi: + write: "true" +nodes: + n01: + profiles: + - default +`, + outDb: `WW_INTERNAL: 43 +nodeprofiles: + default: + ipmi: + write: "true" +nodes: + n01: + profiles: + - default + ipmi: + write: "false" +`} + run_test(t, test) +} + func Test_Multiple_Set_Tests(t *testing.T) { tests := []test_description{ {name: "single node change profile", diff --git a/internal/app/wwctl/profile/add/main_test.go b/internal/app/wwctl/profile/add/main_test.go index 1ef1b875..d44f3c68 100644 --- a/internal/app/wwctl/profile/add/main_test.go +++ b/internal/app/wwctl/profile/add/main_test.go @@ -49,7 +49,7 @@ nodes: {} for _, tt := range tests { env := testenv.New(t) env.WriteFile(t, "etc/warewulf/nodes.conf", - `WW_INTERNAL: 43`) + `WW_INTERNAL: 45`) var err error t.Run(tt.name, func(t *testing.T) { baseCmd := GetCommand() @@ -66,7 +66,7 @@ nodes: {} config, configErr := node.New() assert.NoError(t, configErr) dumpBytes, _ := config.Dump() - assert.Equal(t, tt.outDb, string(dumpBytes)) + assert.YAMLEq(t, tt.outDb, string(dumpBytes)) }) } } diff --git a/internal/pkg/api/apiconfig/client_server.go b/internal/pkg/api/apiconfig/client_server.go index c227e224..9cbd168e 100644 --- a/internal/pkg/api/apiconfig/client_server.go +++ b/internal/pkg/api/apiconfig/client_server.go @@ -4,7 +4,7 @@ import ( "log" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // ClientServerConfig is the full client server configuration. diff --git a/internal/pkg/api/node/set.go b/internal/pkg/api/node/set.go index 6d539200..9c212e47 100644 --- a/internal/pkg/api/node/set.go +++ b/internal/pkg/api/node/set.go @@ -43,7 +43,7 @@ func NodeSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodeYaml, } //func AbstractSetParameterCheck(set *wwapiv1.ConfSetParameter, confMap map[string]*node.NodeConf, confs []string) (count uint, err error) { if set == nil { - err = fmt.Errorf("profile set parameter is nil") + err = fmt.Errorf("node set parameter is nil") return } if set.ConfList == nil { diff --git a/internal/pkg/api/profile/set.go b/internal/pkg/api/profile/set.go index 86f6e108..958e8400 100644 --- a/internal/pkg/api/profile/set.go +++ b/internal/pkg/api/profile/set.go @@ -3,6 +3,8 @@ package apiprofile import ( "fmt" + "dario.cat/mergo" + "github.com/pkg/errors" "github.com/warewulf/warewulf/internal/pkg/api/routes/wwapiv1" "github.com/warewulf/warewulf/internal/pkg/node" @@ -65,10 +67,17 @@ func ProfileSetParameterCheck(set *wwapiv1.ConfSetParameter) (nodeDB node.NodeYa wwlog.Warn("invalid profile: %s", profileId) continue } - err = yaml.Unmarshal([]byte(set.NodeConfYaml), profilePtr) + newProfile := node.EmptyProfile() + err = yaml.Unmarshal([]byte(set.NodeConfYaml), &newProfile) if err != nil { return } + // merge in + err = mergo.Merge(profilePtr, &newProfile, mergo.WithOverride) + if err != nil { + return + } + if set.NetdevDelete != "" { if _, ok := profilePtr.NetDevs[set.NetdevDelete]; !ok { err = fmt.Errorf("network device name doesn't exist: %s", set.NetdevDelete) diff --git a/internal/pkg/node/constructors.go b/internal/pkg/node/constructors.go index 4cec76c1..148983a2 100644 --- a/internal/pkg/node/constructors.go +++ b/internal/pkg/node/constructors.go @@ -1,9 +1,10 @@ package node import ( + "bytes" + "encoding/gob" "os" "path" - "reflect" "sort" "dario.cat/mergo" @@ -58,15 +59,28 @@ func Parse(data []byte) (nodeList NodeYaml, err error) { Get a node with its merged in nodes */ func (config *NodeYaml) GetNode(id string) (node NodeConf, err error) { - if _, ok := config.nodes[id]; !ok { return node, ErrNotFound } - node = *config.nodes[id] + node = EmptyNode() + // create a deep copy of the node, as otherwise pointers + // and not their contents is merged + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err = enc.Encode(config.nodes[id]) + if err != nil { + return node, err + } + err = dec.Decode(&node) + if err != nil { + return node, err + } for _, p := range cleanList(config.nodes[id].Profiles) { includedProfile, err := config.GetProfile(p) if err != nil { - return node, err + wwlog.Warn("profile not found: %s", p) + continue } err = mergo.Merge(&node.ProfileConf, includedProfile, mergo.WithAppendSlice) if err != nil { @@ -76,9 +90,6 @@ func (config *NodeYaml) GetNode(id string) (node NodeConf, err error) { // err = mergo.Merge(&node, config.nodes[id], mergo.WithOverride, mergo.WithoutDereference) // err = mergo.Merge(&node, config.nodes[id], mergo.WithOverride) // err = mergo.Merge(&node, config.nodes[id]) - if err != nil { - return node, err - } /* node = EmptyNode() var buf bytes.Buffer @@ -266,7 +277,7 @@ func (config *NodeYaml) ListAllProfiles() []string { /* FindDiscoverableNode returns the first discoverable node and an -interface to associate with the discovered interface. If the node has +interface to associate with the discovered interface. If the nodUNDEFe has a primary interface, it is returned; otherwise, the first interface without a hardware address is returned. @@ -277,7 +288,7 @@ func (config *NodeYaml) FindDiscoverableNode() (NodeConf, string, error) { nodes, _ := config.FindAllNodes() for _, node := range nodes { - if !node.Discoverable { + if !(node.Discoverable.Bool()) { continue } if _, ok := node.NetDevs[node.PrimaryNetDev]; ok { @@ -292,30 +303,3 @@ func (config *NodeYaml) FindDiscoverableNode() (NodeConf, string, error) { return EmptyNode(), "", ErrNoUnconfigured } -func appendStringSlices(src, dst any) { - //srcType := reflect.TypeOf(src) - srcVal := reflect.ValueOf(src) - dstType := reflect.TypeOf(dst) - dstVal := reflect.ValueOf(dst) - for i := 0; i < dstType.Elem().NumField(); i++ { - // wwlog.Debug("dstType.Name: %s", dstType.Elem().Field(i).Name) - srcValField := srcVal.Elem().FieldByName(dstType.Elem().Field(i).Name) - if !srcValField.IsZero() { - if srcValField.Type() == reflect.TypeOf([]string{}) { - // wwlog.Debug("dstType.Name: %s", dstType.Elem().Field(i).Name) - if srcValField.Len() > 0 { - // wwlog.Debug("srcValField.Len(): %d", srcValField.Len()) - for _, elem := range srcValField.Interface().([]string) { - dstVal.Elem().Field(i).Set(reflect.Append(dstVal.Elem().Field(i), reflect.ValueOf(elem))) - wwlog.Debug("elem: %s", elem) - } - lst := dstVal.Elem().Field(i).Addr().Interface().(*[]string) - *lst = cleanList(*lst) - } - - } - } else if srcValField.Type().Kind() == reflect.Ptr { - appendStringSlices(srcValField, dstVal.Elem().Field(i).Interface()) - } - } -} diff --git a/internal/pkg/node/constuctors_test.go b/internal/pkg/node/constuctors_test.go index 4e51a943..6cb64ff3 100644 --- a/internal/pkg/node/constuctors_test.go +++ b/internal/pkg/node/constuctors_test.go @@ -3,16 +3,21 @@ package node import ( "testing" - "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "github.com/warewulf/warewulf/internal/pkg/wwlog" + "gopkg.in/yaml.v3" ) -func newConstructorPrimaryNetworkTest() NodeYaml { +func newConstructorPrimaryNetworkTest(t *testing.T) NodeYaml { var data = ` nodeprofiles: default: comment: This profile is automatically included for each node + overrideprofile: + network devices: + override: + device: ib0 + type: profile nodes: test_node1: network devices: @@ -38,15 +43,25 @@ nodes: device: eth0 net1: device: eth1 + test_node5: + profiles: + - overrideprofile + test_node6: + profiles: + - overrideprofile + network devices: + override: + device: ib1 ` var ret NodeYaml - _ = yaml.Unmarshal([]byte(data), &ret) + err := yaml.Unmarshal([]byte(data), &ret) + assert.NoError(t, err) return ret } func Test_Primary_Network(t *testing.T) { - wwlog.SetLogLevel(wwlog.DEBUG) - c := newConstructorPrimaryNetworkTest() + //wwlog.SetLogLevel(wwlog.DEBUG) + c := newConstructorPrimaryNetworkTest(t) test_node1, err := c.GetNode("test_node1") assert.NoError(t, err) test_node2, err := c.GetNode("test_node2") @@ -55,6 +70,10 @@ func Test_Primary_Network(t *testing.T) { assert.NoError(t, err) test_node4, err := c.GetNode("test_node4") assert.NoError(t, err) + test_node5, err := c.GetNode("test_node5") + assert.NoError(t, err) + test_node6, err := c.GetNode("test_node6") + assert.NoError(t, err) t.Run("Primary network with one network, nothing set", func(t *testing.T) { if test_node1.PrimaryNetDev != "net0" { t.Errorf("primary network isn't net0 but: %s", test_node1.PrimaryNetDev) @@ -92,6 +111,14 @@ func Test_Primary_Network(t *testing.T) { t.Errorf("node primary flag isn't set") } }) + t.Run("defined in profile", func(t *testing.T) { + assert.Equal(t, test_node5.NetDevs["override"].Device, "ib0") + assert.Equal(t, test_node5.NetDevs["override"].Type, "profile") + }) + t.Run("redefined in profile", func(t *testing.T) { + assert.Equal(t, test_node6.NetDevs["override"].Device, "ib1") + assert.Equal(t, test_node6.NetDevs["override"].Type, "profile") + }) } var findDiscoverableNodeTests = []struct { @@ -110,16 +137,16 @@ var findDiscoverableNodeTests = []struct { func Test_FindDiscoverableNode(t *testing.T) { for _, tt := range findDiscoverableNodeTests { t.Run(tt.description, func(t *testing.T) { - config := newConstructorPrimaryNetworkTest() + config := newConstructorPrimaryNetworkTest(t) for _, node := range tt.discoverable_nodes { - config.Nodes[node].Discoverable = true + config.nodes[node].Discoverable = "true" } discovered_node, discovered_interface, err := config.FindDiscoverableNode() if !tt.succeed { assert.Error(t, err) } else { assert.NoError(t, err) - assert.Equal(t, tt.discovered_node, discovered_node) + assert.Equal(t, tt.discovered_node, discovered_node.Id()) assert.Equal(t, tt.discovered_interface, discovered_interface) } }) @@ -147,8 +174,8 @@ nodes: - profile2 node3: runtime overlay: - - n1o1 - - n1o2 + - n3o1 + - n3o2 profiles: - profile1 node4: @@ -170,27 +197,24 @@ nodes: var ymlSrc NodeYaml err := yaml.Unmarshal([]byte(nodesconf), &ymlSrc) assert.NoError(err) + wwlog.SetLogLevel(wwlog.DEBUG) nodes, err := ymlSrc.FindAllNodes() assert.NoError(err) - nodemap := make(map[string]*NodeInfo) + nodemap := make(map[string]*NodeConf) for i := range nodes { - nodemap[nodes[i].Id.Get()] = &nodes[i] + nodemap[nodes[i].Id()] = &nodes[i] } assert.Contains(nodemap, "node1") - assert.ElementsMatch(nodemap["node1"].RuntimeOverlay.GetSlice(), []string{"p1o1", "p1o2"}) - assert.Equal("p1o1,p1o2", nodemap["node1"].RuntimeOverlay.Print()) + + assert.ElementsMatch(nodemap["node1"].RuntimeOverlay, []string{"p1o1", "p1o2"}) assert.Contains(nodemap, "node2") - assert.ElementsMatch(nodemap["node2"].RuntimeOverlay.GetSlice(), []string{"p1o1", "p1o2", "p2o1", "p2o2"}) - assert.Equal("p1o1,p1o2,p2o1,p2o2", nodemap["node2"].RuntimeOverlay.Print()) + assert.ElementsMatch(nodemap["node2"].RuntimeOverlay, []string{"p1o1", "p1o2", "p2o1", "p2o2"}) assert.Contains(nodemap, "node3") - assert.ElementsMatch(nodemap["node3"].RuntimeOverlay.GetSlice(), []string{"p1o1", "p1o2", "n1o1", "n1o2"}) - assert.Equal("p1o1,p1o2,n1o1,n1o2", nodemap["node3"].RuntimeOverlay.Print()) + assert.ElementsMatch(nodemap["node3"].RuntimeOverlay, []string{"p1o1", "p1o2", "n3o1", "n3o2"}) assert.Contains(nodemap, "node4") - assert.ElementsMatch(nodemap["node4"].RuntimeOverlay.GetSlice(), []string{"p1o1", "p1o2", "p2o1", "p2o2", "n1o1", "n1o2"}) - assert.Equal("p1o1,p1o2,p2o1,p2o2,n1o1,n1o2", nodemap["node4"].RuntimeOverlay.Print()) + assert.ElementsMatch(nodemap["node4"].RuntimeOverlay, []string{"p1o1", "p1o2", "p2o1", "p2o2", "n1o1", "n1o2"}) assert.Contains(nodemap, "node5") - assert.ElementsMatch(nodemap["node5"].RuntimeOverlay.GetSlice(), []string{"p1o1", "p2o1", "p2o2", "n1o1"}) - assert.Equal("p1o1,p2o1,p2o2,n1o1 ~{p1o2}", nodemap["node5"].RuntimeOverlay.Print()) + assert.ElementsMatch(nodemap["node5"].RuntimeOverlay, []string{"p1o1", "p1o2", "~p1o2", "p2o1", "p2o2", "n1o1"}) } func Test_negated_list(t *testing.T) { diff --git a/internal/pkg/node/datastructure.go b/internal/pkg/node/datastructure.go index c0914d27..fe47fae9 100644 --- a/internal/pkg/node/datastructure.go +++ b/internal/pkg/node/datastructure.go @@ -18,9 +18,6 @@ Structure of which goes to disk */ type NodeYaml struct { WWInternal int `yaml:"WW_INTERNAL,omitempty" json:"WW_INTERNAL,omitempty"` - NodeProfiles map[string]*ProfileConf - Nodes map[string]*NodeConf - WWInternal int `yaml:"WW_INTERNAL"` nodeProfiles map[string]*ProfileConf nodes map[string]*NodeConf } @@ -32,7 +29,7 @@ type NodeConf struct { id string valid bool // Is set true, if called by the constructor // exported values - Discoverable bool `yaml:"discoverable,omitempty" lopt:"discoverable" sopt:"e" comment:"Make discoverable in given network (true/false)"` + Discoverable wwtype.WWbool `yaml:"discoverable,omitempty" lopt:"discoverable" sopt:"e" comment:"Make discoverable in given network (true/false)"` AssetKey string `yaml:"asset key,omitempty" lopt:"asset" comment:"Set the node's Asset tag (key)"` Profiles []string `yaml:"profiles,omitempty" lopt:"profile" sopt:"P" comment:"Set the node's profile members (comma separated)"` ProfileConf `yaml:"-,inline"` // include all values set in the profile, but inline them in yaml output if these are part of NodeConf diff --git a/internal/pkg/node/flags.go b/internal/pkg/node/flags.go index aae6c332..04e97476 100644 --- a/internal/pkg/node/flags.go +++ b/internal/pkg/node/flags.go @@ -83,8 +83,9 @@ Helper function to create the different PersistentFlags() for different types. */ func createFlags(baseCmd *cobra.Command, myType reflect.StructField, myVal *reflect.Value) { + var wwbool wwtype.WWbool if myType.Tag.Get("lopt") != "" { - if myType.Type.Kind() == reflect.String { + if myType.Type == reflect.TypeOf("") { ptr := myVal.Addr().Interface().(*string) if myType.Tag.Get("sopt") != "" { baseCmd.PersistentFlags().StringVarP(ptr, @@ -128,21 +129,6 @@ func createFlags(baseCmd *cobra.Command, map[string]string{}, // empty default! myType.Tag.Get("comment")) } - } else if myType.Type == reflect.TypeOf(true) { - ptr := myVal.Addr().Interface().(*bool) - if myType.Tag.Get("sopt") != "" { - baseCmd.PersistentFlags().BoolVarP(ptr, - myType.Tag.Get("lopt"), - myType.Tag.Get("sopt"), - false, // empty default! - myType.Tag.Get("comment")) - } else { - baseCmd.PersistentFlags().BoolVar(ptr, - myType.Tag.Get("lopt"), - false, // empty default! - myType.Tag.Get("comment")) - } - } else if myType.Type == reflect.TypeOf(true) { ptr := myVal.Addr().Interface().(*bool) if myType.Tag.Get("sopt") != "" { @@ -185,7 +171,7 @@ func createFlags(baseCmd *cobra.Command, net.IPMask{}, // empty default! myType.Tag.Get("comment")) } - } else if myType.Type == reflect.TypeOf(wwtype.WWbool{}) { + } else if myType.Type == reflect.TypeOf(wwbool) { ptr := myVal.Addr().Interface().(*wwtype.WWbool) if myType.Tag.Get("sopt") != "" { baseCmd.PersistentFlags().VarP(ptr, diff --git a/internal/pkg/node/hash_test.go b/internal/pkg/node/hash_test.go index b2e84d7e..482a7ef8 100644 --- a/internal/pkg/node/hash_test.go +++ b/internal/pkg/node/hash_test.go @@ -3,7 +3,7 @@ package node import ( "testing" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/stretchr/testify/assert" ) diff --git a/internal/pkg/node/list.go b/internal/pkg/node/list.go index b63c3cf8..b82bf246 100644 --- a/internal/pkg/node/list.go +++ b/internal/pkg/node/list.go @@ -1,9 +1,10 @@ package node import ( - "fmt" "net" "reflect" + "sort" + "strconv" "strings" ) @@ -16,34 +17,63 @@ type NodeFields struct { Value string } -type fieldMap map[string]NodeFields +func (f *NodeFields) Set(src, val string) { + if val == "" { + return + } + if f.Value == "" { + f.Value = val + f.Source = src + } else if f.Source != "" { + f.Value = val + if src == "" { + f.Source = "SUPERSEDED" + } else { + f.Source = src + } + } + +} + +type fieldMap map[string]*NodeFields /* Get all the info out of NodeConf. If emptyFields is set true, all fields are shown not only the ones with effective values */ -func (nodeYml *NodeYaml) GetFields(node NodeConf, emptyFields bool) (output []NodeFields) { +func (nodeYml *NodeYaml) GetFields(node NodeConf) (output []NodeFields) { nodeMap := make(fieldMap) for _, p := range node.Profiles { if profile, ok := nodeYml.nodeProfiles[p]; ok { - nodeMap.recursiveFields(profile, emptyFields, "", p) + nodeMap.recursiveFields(profile, "", p) } } - nodeMap.recursiveFields(&node, emptyFields, "", "") + rawNode, _ := nodeYml.GetNodeOnlyPtr(node.id) + nodeMap.recursiveFields(rawNode, "", "") for _, elem := range nodeMap { - output = append(output, elem) + if elem.Value != "" { + output = append(output, *elem) + } } + sort.Slice(output, func(i, j int) bool { + return output[i].Field < output[j].Field + }) return output } /* Get all the info out of ProfileConf. If emptyFields is set true, all fields are shown not only the ones with effective values */ -func (nodeYml *NodeYaml) GetFieldsProfile(profile ProfileConf, emptyFields bool) (output []NodeFields) { +func (nodeYml *NodeYaml) GetFieldsProfile(profile ProfileConf) (output []NodeFields) { profileMap := make(fieldMap) - profileMap.recursiveFields(&profile, emptyFields, "", "") + profileMap.recursiveFields(&profile, "", "") for _, elem := range profileMap { - output = append(output, elem) + if elem.Value != "" { + output = append(output, *elem) + } } + sort.Slice(output, func(i, j int) bool { + return output[i].Field < output[j].Field + }) return output } @@ -51,52 +81,68 @@ func (nodeYml *NodeYaml) GetFieldsProfile(profile ProfileConf, emptyFields bool) Internal function which travels through all fields of a NodeConf and for this reason needs to be called via interface{} */ -func (fieldMap *fieldMap) recursiveFields(obj interface{}, emptyFields bool, prefix string, source string) { +func (fieldMap fieldMap) recursiveFields(obj interface{}, prefix string, source string) { valObj := reflect.ValueOf(obj) typeObj := reflect.TypeOf(obj) + if valObj.IsNil() { + return + } for i := 0; i < typeObj.Elem().NumField(); i++ { - fmt.Printf("name: %s\n", typeObj.Elem().Field(i).Name) if valObj.Elem().Field(i).IsValid() { if !typeObj.Elem().Field(i).IsExported() { continue } - if valObj.Elem().Field(i).Kind() == reflect.String && valObj.Elem().Field(i).String() != "" { - fmt.Printf("string: %s\n", valObj.Elem().Field(i).String()) - (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ - Field: prefix + typeObj.Elem().Field(i).Name, - Source: source, - Value: valObj.Elem().Field(i).String(), - } - } else if emptyFields { - (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ - Field: prefix + typeObj.Elem().Field(i).Name + "[]", - Source: source, - } - } else if typeObj.Elem().Field(i).Type == reflect.TypeOf([]string{}) && valObj.Elem().Field(i).Len() != 0 { - vals := (valObj.Elem().Field(i).Interface()).([]string) - (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ - Field: prefix + typeObj.Elem().Field(i).Name, - Source: source, - Value: strings.Join(vals, ","), - } - } else if typeObj.Elem().Field(i).Type == reflect.TypeOf(net.IP{}) { - val := (valObj.Elem().Field(i).Interface()).(net.IP) - (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ - Field: prefix + typeObj.Elem().Field(i).Name, - Source: source, - Value: val.String(), - } - } else if typeObj.Elem().Field(i).Type.Kind() == reflect.Map { + switch typeObj.Elem().Field(i).Type.Kind() { + case reflect.Map: mapIter := valObj.Elem().Field(i).MapRange() for mapIter.Next() { - fieldMap.recursiveFields(mapIter.Value().Interface(), - emptyFields, prefix+typeObj.Elem().Field(i).Name+"["+mapIter.Key().String()+"].", source) + fieldMap.recursiveFields(mapIter.Value().Interface(), prefix+typeObj.Elem().Field(i).Name+"["+mapIter.Key().String()+"].", source) } - if valObj.Elem().Field(i).Len() == 0 && emptyFields { - (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + if valObj.Elem().Field(i).Len() == 0 { + fieldMap[prefix+typeObj.Elem().Field(i).Name] = &NodeFields{ Field: prefix + typeObj.Elem().Field(i).Name + "[]", } } + case reflect.Struct: + fieldMap.recursiveFields(valObj.Elem().Field(i).Addr().Interface(), "", source) + case reflect.Ptr: + if valObj.Elem().Field(i).Addr().IsValid() { + fieldMap.recursiveFields(valObj.Elem().Field(i).Interface(), prefix+typeObj.Elem().Field(i).Name+".", source) + } + default: + if _, ok := fieldMap[prefix+typeObj.Elem().Field(i).Name]; !ok { + fieldMap[prefix+typeObj.Elem().Field(i).Name] = &NodeFields{ + Field: prefix + typeObj.Elem().Field(i).Name, + Source: source, + } + } + + switch typeObj.Elem().Field(i).Type { + case reflect.TypeOf([]string{}): + vals := (valObj.Elem().Field(i).Interface()).([]string) + fieldMap[prefix+typeObj.Elem().Field(i).Name] = &NodeFields{ + Field: prefix + typeObj.Elem().Field(i).Name, + Source: source, + Value: strings.Join(vals, ","), + } + case reflect.TypeOf(net.IP{}): + val := (valObj.Elem().Field(i).Interface()).(net.IP) + if val != nil { + fieldMap[prefix+typeObj.Elem().Field(i).Name].Set(source, val.String()) + } + case reflect.TypeOf(net.IPMask{}): + val := (valObj.Elem().Field(i).Interface()).(net.IPMask) + if val != nil { + fieldMap[prefix+typeObj.Elem().Field(i).Name].Set(source, val.String()) + } + case reflect.TypeOf(true): + val := (valObj.Elem().Field(i).Interface()).(bool) + if val { + fieldMap[prefix+typeObj.Elem().Field(i).Name].Set(source, strconv.FormatBool(val)) + } + default: + fieldMap[prefix+typeObj.Elem().Field(i).Name].Set(source, valObj.Elem().Field(i).String()) + } } /*else if typeObj.Elem().Field(i).Type.Kind() == reflect.Ptr { fieldMap.recursiveFields(valObj.Elem().Field(i).Interface(), emptyFields, prefix+typeObj.Elem().Field(i).Name+".", source) diff --git a/internal/pkg/node/methods.go b/internal/pkg/node/methods.go index 29e73eb6..72929a42 100644 --- a/internal/pkg/node/methods.go +++ b/internal/pkg/node/methods.go @@ -1,8 +1,6 @@ package node import ( - "bytes" - "encoding/gob" "net" "reflect" "regexp" @@ -10,7 +8,6 @@ import ( "strings" "github.com/warewulf/warewulf/internal/pkg/util" - "github.com/warewulf/warewulf/internal/pkg/wwtype" ) type sortByName []NodeConf @@ -134,51 +131,92 @@ func (info *ProfileConf) Flatten() { recursiveFlatten(info) } -// abstract flatten -func recursiveFlatten(strct interface{}) { - confType := reflect.TypeOf(strct) - confVal := reflect.ValueOf(strct) - for j := 0; j < confType.Elem().NumField(); j++ { - if confVal.Elem().Field(j).Type().Kind() == reflect.Ptr && !confVal.Elem().Field(j).IsNil() { - // iterate now over the ptr fields - setToNil := true - nestedType := reflect.TypeOf(confVal.Elem().Field(j).Interface()) - nestedVal := reflect.ValueOf(confVal.Elem().Field(j).Interface()) - for i := 0; i < nestedType.Elem().NumField(); i++ { - // wwlog.Debug("checking %s", nestedType.Elem().Field(i).Type.String()) - if nestedType.Elem().Field(i).Type.Kind() == reflect.String && - nestedVal.Elem().Field(i).Interface().(string) != "" && - nestedVal.Elem().Field(i).Interface().(string) != undef { - setToNil = false - } else if nestedType.Elem().Field(i).Type == reflect.TypeOf([]string{}) && - len(nestedVal.Elem().Field(i).Interface().([]string)) != 0 { - setToNil = false - } else if nestedType.Elem().Field(i).Type == reflect.TypeOf(map[string]string{}) && - len(nestedVal.Elem().Field(i).Interface().(map[string]string)) != 0 { - setToNil = false - } else if nestedType.Elem().Field(i).Type == reflect.TypeOf(net.IP{}) { - val := nestedVal.Elem().Field(i).Interface().(net.IP) - if len(val) != 0 && !val.IsUnspecified() { - setToNil = false - } - } else if nestedType.Elem().Field(i).Type == reflect.TypeOf(wwtype.WWbool{}) { - val := nestedVal.Elem().Field(i).Interface().(wwtype.WWbool) - if !val.IsZero() { - setToNil = false - } - } - } - if setToNil { - confVal.Elem().Field(j).Set(reflect.Zero(confVal.Elem().Field(j).Type())) - } - } else if confType.Elem().Field(j).Anonymous { - recursiveFlatten(confVal.Elem().Field(j).Addr().Interface()) - } else if confType.Elem().Field(j).IsExported() && confType.Elem().Field(j).Type.Kind() == reflect.String { - if confVal.Elem().Field(j).Interface().(string) == undef { - confVal.Elem().Field(j).SetString("") +func recursiveFlatten(obj interface{}) (hasContent bool) { + valObj := reflect.ValueOf(obj) + typeObj := reflect.TypeOf(obj) + hasContent = false + if valObj.IsNil() { + return + } + for i := 0; i < typeObj.Elem().NumField(); i++ { + if valObj.Elem().Field(i).IsValid() { + if !typeObj.Elem().Field(i).IsExported() { + continue } } + switch typeObj.Elem().Field(i).Type.Kind() { + case reflect.Map: + mapIter := valObj.Elem().Field(i).MapRange() + for mapIter.Next() { + if mapIter.Value().Kind() == reflect.String { + if mapIter.Value().String() != "" { + // fmt.Println("map") + hasContent = true + } + } else { + ret := recursiveFlatten(mapIter.Value().Interface()) + hasContent = ret || hasContent + } + } + + case reflect.Ptr: + if valObj.Elem().Field(i).Addr().IsValid() { + ret := recursiveFlatten((valObj.Elem().Field(i).Interface())) + if !ret { + valObj.Elem().Field(i).Set(reflect.Zero(valObj.Elem().Field(i).Type())) + } + hasContent = ret || hasContent + + } + case reflect.Struct: + ret := recursiveFlatten((valObj.Elem().Field(i).Addr().Interface())) + hasContent = ret || hasContent + case reflect.Slice: + if typeObj.Elem().Field(i).Type == reflect.TypeOf([]string{}) { + del := false + for _, elem := range (valObj.Elem().Field(i).Interface()).([]string) { + if strings.EqualFold(elem, undef) { + del = true + } + } + if del { + valObj.Elem().Field(i).SetLen(0) + } + } + if valObj.Elem().Field(i).Len() > 0 { + hasContent = true + } + case reflect.String: + if strings.EqualFold(valObj.Elem().Field(i).String(), undef) { + valObj.Elem().Field(i).SetString("") + } + if valObj.Elem().Field(i).String() != "" { + hasContent = true + } + case reflect.Bool: + val := valObj.Elem().Field(i).Interface().(bool) + hasContent = hasContent || val + default: + switch valObj.Elem().Field(i).Type() { + case reflect.TypeOf(net.IP{}): + val := valObj.Elem().Field(i).Interface().(net.IP) + if len(val) != 0 && !val.IsUnspecified() { + hasContent = true + } + case reflect.TypeOf(net.IPMask{}): + val := valObj.Elem().Field(i).Interface().(net.IPMask) + if len(val) != 0 { + // fmt.Println("Mask") + hasContent = true + } + default: + } + } + if !hasContent { + valObj.Elem().Field(i).Set(reflect.Zero(valObj.Elem().Field(i).Type())) + } } + return } /* @@ -308,38 +346,3 @@ func cleanList(list []string) (ret []string) { } return ret } - -// Clone (deep copy) a node via gob -func (src *NodeConf) Clone() (dst NodeConf, err error) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - dec := gob.NewDecoder(&buf) - err = enc.Encode(src) - if err != nil { - return - } - err = dec.Decode(&dst) - if err != nil { - return - } - dst.id = src.id - dst.valid = src.valid - return -} - -// Clone (deep copy) a node via gob -func (src ProfileConf) Clone() (dst ProfileConf, err error) { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - dec := gob.NewDecoder(&buf) - err = enc.Encode(src) - if err != nil { - return - } - err = dec.Decode(&dst) - if err != nil { - return - } - dst.id = src.id - return -} diff --git a/internal/pkg/node/modifiers.go b/internal/pkg/node/modifiers.go index a7f46736..0efef3ff 100644 --- a/internal/pkg/node/modifiers.go +++ b/internal/pkg/node/modifiers.go @@ -112,12 +112,12 @@ func (config *NodeYaml) Persist() error { out, dumpErr := config.Dump() if dumpErr != nil { wwlog.Error("%s", dumpErr) - os.Exit(1) + return dumpErr } file, err := os.OpenFile(ConfigFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { wwlog.Error("%s", err) - os.Exit(1) + return err } defer file.Close() _, err = file.WriteString(string(out)) diff --git a/internal/pkg/node/util_test.go b/internal/pkg/node/util_test.go index eec0aecf..54f2dbbc 100644 --- a/internal/pkg/node/util_test.go +++ b/internal/pkg/node/util_test.go @@ -3,9 +3,9 @@ package node import ( "testing" - "github.com/hpcng/warewulf/internal/pkg/wwlog" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v2" + "github.com/warewulf/warewulf/internal/pkg/wwlog" + "gopkg.in/yaml.v3" ) func NewUtilTestNode() (NodeYaml, error) { diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index 054b19ae..4c36394f 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -41,7 +41,7 @@ func loadNodeDB() (err error) { } for _, n := range nodes { - if n.Discoverable { + if n.Discoverable.Bool() { continue } for _, netdev := range n.NetDevs { @@ -68,7 +68,7 @@ func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { } // If we failed to find a node, let's see if we can add one... - wwlog.Warn("(node not configured)", hwaddr) + wwlog.Warn("node not configured: %s", hwaddr) node, netdev, err := db.yml.FindDiscoverableNode() if err != nil { @@ -77,11 +77,12 @@ func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { return node, err } // update node - nodeChanges, _ := db.yml.GetNodeOnly(nId) // ignore error as nodeId is in db + wwlog.Debug("discoverd node: %s netdev: %s", node.Id(), netdev) + nodeChanges, _ := db.yml.GetNodeOnly(node.Id()) // ignore error as nodeId is in db wwlog.Debug("node: %v", nodeChanges) nodeChanges.NetDevs[netdev].Hwaddr = hwaddr - nodeChanges.Discoverable = false - err = db.yml.SetNode(nId, nodeChanges) + nodeChanges.Discoverable = "UNDEF" + err = db.yml.SetNode(node.Id(), nodeChanges) if err != nil { return node, err } @@ -97,8 +98,8 @@ func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { // be done automatically when attempting to serve an overlay that // hasn't been built (without blocking the database). - wwlog.Serv("%s (node %s automatically configured)", hwaddr, nId) + wwlog.Serv("%s (node %s automatically configured)", hwaddr, node.Id()) // return the discovered node - return db.yml.GetNode(nId) + return db.yml.GetNode(node.Id()) } diff --git a/internal/pkg/warewulfd/provision_test.go b/internal/pkg/warewulfd/provision_test.go index 7f112553..2f644019 100644 --- a/internal/pkg/warewulfd/provision_test.go +++ b/internal/pkg/warewulfd/provision_test.go @@ -10,10 +10,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/assert" - warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" - "github.com/warewulf/warewulf/internal/pkg/node" + "github.com/warewulf/warewulf/internal/pkg/testenv" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) @@ -38,13 +36,18 @@ var provisionSendTests = []struct { func Test_ProvisionSend(t *testing.T) { env := testenv.New(t) - // wwlog.SetLogLevel(wwlog.DEBUG) - env.WriteFile(t, "etc/warewulf/nodes.conf", ` + + env.WriteFile(t, "etc/warewulf/nodes.conf", `WW_INTERNAL: 45 +nodeprofiles: + default: + container name: suse nodes: n1: network devices: default: hwaddr: 00:00:00:ff:ff:ff + profiles: + - default n2: network devices: default: @@ -59,19 +62,12 @@ nodes: kernel: override: 1.1.1`) // create a arp file as for grub we look up the ip address through the arp cache - arp_file, err := os.CreateTemp(os.TempDir(), "ww-arp") - assert.NoError(t, err) - defer arp_file.Close() - { - _, err := arp_file.WriteString(`IP address HW type Flags HW address Mask Device + + env.WriteFile(t, "arpcache", `IP address HW type Flags HW address Mask Device 10.10.10.10 0x1 0x2 00:00:00:ff:ff:ff * dummy 10.10.10.11 0x1 0x2 00:00:00:00:ff:ff * dummy 10.10.10.12 0x1 0x2 00:00:00:00:00:ff * dummy`) - assert.NoError(t, err) - } - assert.NoError(t, arp_file.Sync()) - SetArpFile(arp_file.Name()) - + SetArpFile(path.Join(env.BaseDir, "arpcache")) conf := warewulfconf.Get() containerDir, imageDirErr := os.MkdirTemp(os.TempDir(), "ww-test-container-*") assert.NoError(t, imageDirErr) @@ -106,18 +102,13 @@ nodes: dbErr := LoadNodeDB() assert.NoError(t, dbErr) - conf := warewulfconf.Get() conf.Warewulf.Secure = false - wwlog.SetLogLevel(wwlog.DEBUG) - assert.NoError(t, os.MkdirAll(path.Join(provisionDir, "overlays", "n1"), 0700)) - assert.NoError(t, os.WriteFile(path.Join(provisionDir, "overlays", "n1", "__SYSTEM__.img"), []byte("system overlay"), 0600)) - assert.NoError(t, os.WriteFile(path.Join(provisionDir, "overlays", "n1", "__RUNTIME__.img"), []byte("runtime overlay"), 0600)) - assert.NoError(t, os.WriteFile(path.Join(provisionDir, "overlays", "n1", "o1.img"), []byte("specific overlay"), 0600)) assert.NoError(t, os.MkdirAll(path.Join(conf.Paths.WWProvisiondir, "overlays", "n1"), 0700)) assert.NoError(t, os.WriteFile(path.Join(conf.Paths.WWProvisiondir, "overlays", "n1", "__SYSTEM__.img"), []byte("system overlay"), 0600)) assert.NoError(t, os.WriteFile(path.Join(conf.Paths.WWProvisiondir, "overlays", "n1", "__RUNTIME__.img"), []byte("runtime overlay"), 0600)) assert.NoError(t, os.WriteFile(path.Join(conf.Paths.WWProvisiondir, "overlays", "n1", "o1.img"), []byte("specific overlay"), 0600)) + wwlog.SetLogLevel(wwlog.DEBUG) for _, tt := range provisionSendTests { t.Run(tt.description, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, tt.url, nil) @@ -129,7 +120,9 @@ nodes: data, readErr := io.ReadAll(res.Body) assert.NoError(t, readErr) - assert.Equal(t, tt.body, string(data)) + if tt.body != "" { + assert.Equal(t, tt.body, string(data)) + } assert.Equal(t, tt.status, res.StatusCode) }) } diff --git a/internal/pkg/wwtype/wwbool.go b/internal/pkg/wwtype/wwbool.go index 078bfb69..650f96a2 100644 --- a/internal/pkg/wwtype/wwbool.go +++ b/internal/pkg/wwtype/wwbool.go @@ -1,86 +1,64 @@ package wwtype import ( + "fmt" "strconv" "strings" "github.com/warewulf/warewulf/internal/pkg/util" ) +// Simple string which can be converted to bool. Backend storage +// is string for better merging +type WWbool string + /* -Type for holding a simple bool, but which can be set via the UNSET parameter +Transform the underlying string value to bool */ -type WWbool struct { - bool // the actual value - isset bool // only true if set through yaml or set, so that false can go to disk - delnext bool // delete after next iteraion -} - -// Yaml marshaler, calls this to find out, if going to disk -func (b WWbool) IsZero() bool { - return !b.isset -} - -func (b *WWbool) String() string { - return strconv.FormatBool(b.bool) -} - -func (b *WWbool) Set(str string) error { +func (val WWbool) Bool() bool { + str := string(val) if util.InSlice(GetUnsetVerbs(), str) { - b.bool = false - b.isset = true - b.delnext = true + return false + } + if strings.ToLower(str) == "yes" { + return true + } + if strings.ToLower(str) == "no" { + return false + } + bval, _ := strconv.ParseBool(str) + return bval +} + +/* +Set the string, only accept bool values like true, false, but also UNDEF +*/ +func (val *WWbool) Set(str string) error { + if util.InSlice(GetUnsetVerbs(), str) { + // run the unset verb trough, will be filtered out later + *val = WWbool(str) return nil } - - if strings.ToLower(str) == "yes" || str == "" { - b.bool = true - b.isset = true + if strings.ToLower(str) == "yes" { + *val = WWbool("true") return nil } if strings.ToLower(str) == "no" { - b.bool = false - b.isset = true + *val = WWbool("false") return nil } - var err error - b.bool, err = strconv.ParseBool(str) + bval, err := strconv.ParseBool(str) if err == nil { - b.isset = true + *val = WWbool(strconv.FormatBool(bval)) + return nil } - return err + return fmt.Errorf("value for WWbool can't be set from %s", str) } -func (b *WWbool) Type() string { +func (val WWbool) String() string { + return string(val) +} + +func (b WWbool) Type() string { return "WWbool" } - -func (b WWbool) MarshalBinary() (buf []byte, err error) { - strconv.AppendBool(buf, b.bool) - return buf, nil -} - -func (b *WWbool) UnmarshalBinary(data []byte) (err error) { - b.bool, err = strconv.ParseBool(string(data)) - return err -} -func (b WWbool) MarshalText() (buf []byte, err error) { - if b.bool { - buf = append(buf, "true"...) - } else if !b.delnext { - buf = append(buf, "false"...) - } else { - buf = append(buf, "delete"...) - } - return buf, nil -} - -func (b *WWbool) UnmarshalText(data []byte) (err error) { - if strings.EqualFold(string(data), "delete") { - b.isset = false - b.bool = false - return nil - } - b.isset = true - return b.Set(string(data)) -}