From 5bd4fd4712cee1bdda545f5123068c69bfaaded7 Mon Sep 17 00:00:00 2001 From: Christian Goll Date: Wed, 20 Dec 2023 15:50:15 +0100 Subject: [PATCH] use yaml/v3 and don't export Nodes introduced wwbool and don't export Nodes and NodeConfs. This requires new explict Yaml (un)marshaling as the standard marshaller won't touch these fields Signed-off-by: Christian Goll --- internal/app/wwctl/node/set/main.go | 3 +- internal/app/wwctl/profile/set/main.go | 3 +- internal/pkg/config/root.go | 2 +- internal/pkg/node/checkconf.go | 3 +- internal/pkg/node/constructors.go | 239 +++++++++++++++---------- internal/pkg/node/datastructure.go | 69 ++++++- internal/pkg/node/flags.go | 17 ++ internal/pkg/node/hash.go | 6 +- internal/pkg/node/list.go | 85 ++++++--- internal/pkg/node/methods.go | 93 +++++++++- internal/pkg/node/modifiers.go | 70 ++++++-- internal/pkg/warewulfd/nodedb.go | 76 +++----- 12 files changed, 470 insertions(+), 196 deletions(-) diff --git a/internal/app/wwctl/node/set/main.go b/internal/app/wwctl/node/set/main.go index e988cb45..93ade466 100644 --- a/internal/app/wwctl/node/set/main.go +++ b/internal/app/wwctl/node/set/main.go @@ -18,9 +18,10 @@ func CobraRunE(vars *variables) func(cmd *cobra.Command, args []string) (err err return func(cmd *cobra.Command, args []string) error { // remove the default network as the all network values are assigned // to this network - if !node.ObjectIsEmpty(vars.nodeConf.NetDevs["UNDEF"]) { + if !node.ObjectIsEmpty(vars.nodeConf.NetDevs["UNDEF"]) || len(vars.nodeAdd.NetTagsAdd) > 0 { netDev := *vars.nodeConf.NetDevs["UNDEF"] vars.nodeConf.NetDevs[vars.nodeAdd.Net] = &netDev + vars.nodeConf.NetDevs[vars.nodeAdd.Net].Tags = vars.nodeAdd.NetTagsAdd } delete(vars.nodeConf.NetDevs, "UNDEF") if vars.nodeAdd.FsName != "" { diff --git a/internal/app/wwctl/profile/set/main.go b/internal/app/wwctl/profile/set/main.go index a22c58af..17edaafa 100644 --- a/internal/app/wwctl/profile/set/main.go +++ b/internal/app/wwctl/profile/set/main.go @@ -19,9 +19,10 @@ func CobraRunE(vars *variables) func(cmd *cobra.Command, args []string) (err err // remove the default network as the all network values are assigned // to this network - if !node.ObjectIsEmpty(vars.profileConf.NetDevs["UNDEF"]) { + if !node.ObjectIsEmpty(vars.profileConf.NetDevs["UNDEF"]) || len(vars.profileAdd.NetTagsAdd) > 0 { netDev := *vars.profileConf.NetDevs["UNDEF"] vars.profileConf.NetDevs[vars.profileAdd.Net] = &netDev + vars.profileConf.NetDevs[vars.profileAdd.Net].Tags = vars.profileAdd.NetTagsAdd } delete(vars.profileConf.NetDevs, "UNDEF") if vars.fsName != "" { diff --git a/internal/pkg/config/root.go b/internal/pkg/config/root.go index 50b2a134..5e6b1d9d 100644 --- a/internal/pkg/config/root.go +++ b/internal/pkg/config/root.go @@ -17,7 +17,7 @@ import ( "github.com/creasty/defaults" "github.com/warewulf/warewulf/internal/pkg/wwlog" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) var cachedConf RootConf diff --git a/internal/pkg/node/checkconf.go b/internal/pkg/node/checkconf.go index 1910360b..89941a24 100644 --- a/internal/pkg/node/checkconf.go +++ b/internal/pkg/node/checkconf.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/warewulf/warewulf/internal/pkg/util" + "github.com/warewulf/warewulf/internal/pkg/wwtype" ) /* @@ -60,7 +61,7 @@ func (nodeConf *NodeConf) Check() (err error) { } func checker(value string, valType string) (niceValue string, err error) { - if valType == "" || value == "" || util.InSlice(GetUnsetVerbs(), value) { + if valType == "" || value == "" || util.InSlice(wwtype.GetUnsetVerbs(), value) { return "", nil } //wwlog.Debug("checker: %s is %s", value, valType) diff --git a/internal/pkg/node/constructors.go b/internal/pkg/node/constructors.go index e33817e4..4cec76c1 100644 --- a/internal/pkg/node/constructors.go +++ b/internal/pkg/node/constructors.go @@ -1,16 +1,16 @@ package node import ( - "bytes" - "encoding/gob" "os" "path" + "reflect" "sort" + "dario.cat/mergo" warewulfconf "github.com/warewulf/warewulf/internal/pkg/config" "github.com/warewulf/warewulf/internal/pkg/wwlog" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) var ( @@ -37,79 +37,82 @@ func New() (NodeYaml, error) { // document. Passes any errors return from yaml.Unmarshal. Returns an // error if any parsed value is not of a valid type for the given // parameter. -func Parse(data []byte) (NodeYaml, error) { - var nodeList NodeYaml - var err error +func Parse(data []byte) (nodeList NodeYaml, err error) { wwlog.Debug("Unmarshaling the node configuration") err = yaml.Unmarshal(data, &nodeList) if err != nil { return nodeList, err } - wwlog.Debug("Checking profiles for types") - if nodeList.Nodes == nil { - nodeList.Nodes = map[string]*NodeConf{} + wwlog.Debug("Checking nodes for types") + if nodeList.nodes == nil { + nodeList.nodes = map[string]*NodeConf{} } - /* - for nodeName, node := range nodeList.Nodes { - err = node.Check() - if err != nil { - wwlog.Warn("node: %s parsing error: %s", nodeName, err) - return nodeList, err - } - } - */ - if nodeList.NodeProfiles == nil { - nodeList.NodeProfiles = map[string]*ProfileConf{} + if nodeList.nodeProfiles == nil { + nodeList.nodeProfiles = map[string]*ProfileConf{} } - /* - for profileName, profile := range nodeList.NodeProfiles { - err = profile.Check() - if err != nil { - wwlog.Warn("node: %s parsing error: %s", profileName, err) - return nodeList, err - } - } - */ - wwlog.Debug("Returning node object") + wwlog.Debug("returning node object") return nodeList, nil } /* -Get a node with its merged in profiles +Get a node with its merged in nodes */ func (config *NodeYaml) GetNode(id string) (node NodeConf, err error) { - if _, ok := config.Nodes[id]; !ok { + + if _, ok := config.nodes[id]; !ok { return node, ErrNotFound } - wwlog.Debug("constructing node: %s", id) - for _, p := range config.Nodes[id].Profiles { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - dec := gob.NewDecoder(&buf) + node = *config.nodes[id] + for _, p := range cleanList(config.nodes[id].Profiles) { includedProfile, err := config.GetProfile(p) if err != nil { return node, err } - err = enc.Encode(includedProfile) - if err != nil { - return node, err - } - err = dec.Decode(&node) + err = mergo.Merge(&node.ProfileConf, includedProfile, mergo.WithAppendSlice) if err != nil { return node, err } } - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - dec := gob.NewDecoder(&buf) - err = enc.Encode(config.Nodes[id]) - if err != nil { - return node, err - } - err = dec.Decode(&node) + // err = mergo.Merge(&node, config.nodes[id], mergo.WithOverride, mergo.WithoutDereference) + // err = mergo.Merge(&node, config.nodes[id], mergo.WithOverride) + // err = mergo.Merge(&node, config.nodes[id]) if err != nil { return node, err } + /* + node = EmptyNode() + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + includedProfile, err := config.GetProfile(p) + appendStringSlices(&node, &includedProfile) + if err != nil { + return node, err + } + err = enc.Encode(includedProfile) + if err != nil { + return node, err + } + err = dec.Decode(&node) + if err != nil { + return node, err + } + wwlog.Debug("merged in profile: %s", p) + } + var bufNode bytes.Buffer + encNode := gob.NewEncoder(&bufNode) + decNode := gob.NewDecoder(&bufNode) + appendStringSlices(&node, &config.nodes[id].ProfileConf) + + err = encNode.Encode(config.nodes[id]) + if err != nil { + return node, err + } + err = decNode.Decode(&node) + if err != nil { + return node, err + } + */ // finally set no exported values node.id = id node.valid = true @@ -127,46 +130,69 @@ func (config *NodeYaml) GetNode(id string) (node NodeConf, err error) { node.PrimaryNetDev = keys[0] } } + wwlog.Debug("constructed node: %s", id) return } /* -Get the profile with all the profiles merged in +Return the node with the id string without the merged in nodes, return ErrNotFound +otherwise */ -func (config *NodeYaml) GetProfile(id string) (profile NodeConf, err error) { - if _, ok := config.NodeProfiles[id]; !ok { - return profile, ErrNotFound +func (config *NodeYaml) GetNodeOnly(id string) (node NodeConf, err error) { + node = EmptyNode() + if found, ok := config.nodes[id]; ok { + return *found, nil } - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - dec := gob.NewDecoder(&buf) - // finally merge in the real profile - err = enc.Encode(config.NodeProfiles[id]) - if err != nil { - return profile, err - } - err = dec.Decode(&profile) - if err != nil { - return profile, err - } - // finally set no exported values - profile.id = id - return + return node, ErrNotFound } /* -Get the profiles from the loaded configuration. This function also merges -the profiles with the given profiles. +Return pointer to the node with the id string without the merged in nodes, return ErrMotFound +otherwise */ -func (config *NodeYaml) FindAllNodes(profiles ...string) (nodeList []NodeConf, err error) { - if len(profiles) == 0 { - for n := range config.Nodes { - profiles = append(profiles, n) +func (config *NodeYaml) GetNodeOnlyPtr(id string) (*NodeConf, error) { + node := EmptyNode() + if found, ok := config.nodes[id]; ok { + return found, nil + } + return &node, ErrNotFound +} + +/* +Get the profile with id, return ErrNotFound otherwise +*/ +func (config *NodeYaml) GetProfile(id string) (profile ProfileConf, err error) { + if found, ok := config.nodeProfiles[id]; ok { + found.id = id + return *found, nil + } + return profile, ErrNotFound +} + +/* +Get the profile with id, return ErrNotFound otherwise +*/ +func (config *NodeYaml) GetProfilePtr(id string) (profile *ProfileConf, err error) { + if found, ok := config.nodeProfiles[id]; ok { + found.id = id + return found, nil + } + return profile, ErrNotFound +} + +/* +Get the nodes from the loaded configuration. This function also merges +the nodes with the given nodes. +*/ +func (config *NodeYaml) FindAllNodes(nodes ...string) (nodeList []NodeConf, err error) { + if len(nodes) == 0 { + for n := range config.nodes { + nodes = append(nodes, n) } } - wwlog.Debug("Finding profiles: %s", profiles) - for _, profileId := range profiles { - node, err := config.GetNode(profileId) + wwlog.Debug("Finding nodes: %s", nodes) + for _, nodeId := range nodes { + node, err := config.GetNode(nodeId) if err != nil { return nodeList, err } @@ -186,16 +212,16 @@ func (config *NodeYaml) FindAllNodes(profiles ...string) (nodeList []NodeConf, e } /* -Return all profiles as NodeInfo +Return all nodes as ProfileConf */ -func (config *NodeYaml) FindAllProfiles(profiles ...string) (profileList []NodeConf, err error) { - if len(profiles) == 0 { - for n := range config.NodeProfiles { - profiles = append(profiles, n) +func (config *NodeYaml) FindAllProfiles(nodes ...string) (profileList []ProfileConf, err error) { + if len(nodes) == 0 { + for n := range config.nodeProfiles { + nodes = append(nodes, n) } } - wwlog.Debug("Finding profiles: %s", profiles) - for _, profileId := range profiles { + wwlog.Debug("Finding nodes: %s", nodes) + for _, profileId := range nodes { node, err := config.GetProfile(profileId) if err != nil { return profileList, err @@ -220,19 +246,19 @@ func (config *NodeYaml) FindAllProfiles(profiles ...string) (profileList []NodeC Return the names of all available nodes */ func (config *NodeYaml) ListAllNodes() []string { - nodeList := make([]string, len(config.Nodes)) - for name := range config.Nodes { + nodeList := make([]string, len(config.nodes)) + for name := range config.nodes { nodeList = append(nodeList, name) } return nodeList } /* -Return the names of all available profiles +Return the names of all available nodes */ func (config *NodeYaml) ListAllProfiles() []string { var nodeList []string - for name := range config.NodeProfiles { + for name := range config.nodeProfiles { nodeList = append(nodeList, name) } return nodeList @@ -246,7 +272,7 @@ without a hardware address is returned. If no unconfigured node is found, an error is returned. */ -func (config *NodeYaml) FindDiscoverableNode() (string, string, error) { +func (config *NodeYaml) FindDiscoverableNode() (NodeConf, string, error) { nodes, _ := config.FindAllNodes() @@ -255,14 +281,41 @@ func (config *NodeYaml) FindDiscoverableNode() (string, string, error) { continue } if _, ok := node.NetDevs[node.PrimaryNetDev]; ok { - return node.Id(), node.PrimaryNetDev, nil + return node, node.PrimaryNetDev, nil } for netdev, dev := range node.NetDevs { if dev.Hwaddr != "" { - return node.Id(), netdev, nil + return node, netdev, nil } } } - return "", "", ErrNoUnconfigured + return EmptyNode(), "", ErrNoUnconfigured +} +func appendStringSlices(src, dst any) { + //srcType := reflect.TypeOf(src) + srcVal := reflect.ValueOf(src) + dstType := reflect.TypeOf(dst) + dstVal := reflect.ValueOf(dst) + for i := 0; i < dstType.Elem().NumField(); i++ { + // wwlog.Debug("dstType.Name: %s", dstType.Elem().Field(i).Name) + srcValField := srcVal.Elem().FieldByName(dstType.Elem().Field(i).Name) + if !srcValField.IsZero() { + if srcValField.Type() == reflect.TypeOf([]string{}) { + // wwlog.Debug("dstType.Name: %s", dstType.Elem().Field(i).Name) + if srcValField.Len() > 0 { + // wwlog.Debug("srcValField.Len(): %d", srcValField.Len()) + for _, elem := range srcValField.Interface().([]string) { + dstVal.Elem().Field(i).Set(reflect.Append(dstVal.Elem().Field(i), reflect.ValueOf(elem))) + wwlog.Debug("elem: %s", elem) + } + lst := dstVal.Elem().Field(i).Addr().Interface().(*[]string) + *lst = cleanList(*lst) + } + + } + } else if srcValField.Type().Kind() == reflect.Ptr { + appendStringSlices(srcValField, dstVal.Elem().Field(i).Interface()) + } + } } diff --git a/internal/pkg/node/datastructure.go b/internal/pkg/node/datastructure.go index 4629ddf8..c0914d27 100644 --- a/internal/pkg/node/datastructure.go +++ b/internal/pkg/node/datastructure.go @@ -1,6 +1,12 @@ package node -import "net" +import ( + "net" + + "github.com/warewulf/warewulf/internal/pkg/wwlog" + "github.com/warewulf/warewulf/internal/pkg/wwtype" + "gopkg.in/yaml.v3" +) const undef string = "UNDEF" @@ -14,14 +20,18 @@ type NodeYaml struct { WWInternal int `yaml:"WW_INTERNAL,omitempty" json:"WW_INTERNAL,omitempty"` NodeProfiles map[string]*ProfileConf Nodes map[string]*NodeConf + WWInternal int `yaml:"WW_INTERNAL"` + nodeProfiles map[string]*ProfileConf + nodes map[string]*NodeConf } /* NodeConf is the datastructure describing a node and a profile which in disk format. */ type NodeConf struct { - id string - valid bool // Is set true, if called by the constructor + id string + valid bool // Is set true, if called by the constructor + // exported values Discoverable bool `yaml:"discoverable,omitempty" lopt:"discoverable" sopt:"e" comment:"Make discoverable in given network (true/false)"` AssetKey string `yaml:"asset key,omitempty" lopt:"asset" comment:"Set the node's Asset tag (key)"` Profiles []string `yaml:"profiles,omitempty" lopt:"profile" sopt:"P" comment:"Set the node's profile members (comma separated)"` @@ -32,6 +42,7 @@ type NodeConf struct { Holds the data which can be set for profiles and nodes. */ type ProfileConf struct { + id string // exported values Comment string `yaml:"comment,omitempty" lopt:"comment" comment:"Set arbitrary string comment"` ClusterName string `yaml:"cluster name,omitempty" lopt:"cluster" sopt:"c" comment:"Set cluster group"` @@ -59,9 +70,10 @@ type IpmiConf struct { Port string `yaml:"port,omitempty" lopt:"ipmiport" comment:"Set the IPMI port"` Interface string `yaml:"interface,omitempty" lopt:"ipmiinterface" comment:"Set the node's IPMI interface (defaults: 'lan')"` EscapeChar string `yaml:"escapechar,omitempty" lopt:"ipmiescapechar" comment:"Set the IPMI escape character (defaults: '~')"` - Write bool `yaml:"write,omitempty" lopt:"ipmiwrite" comment:"Enable the write of impi configuration (true/false)"` + Write wwtype.WWbool `yaml:"write,omitempty" lopt:"ipmiwrite" comment:"Enable the write of impi configuration (true/false)"` Tags map[string]string `yaml:"tags,omitempty"` } + type KernelConf struct { Version string `yaml:"version,omitempty" json:"version,omitempty"` Override string `yaml:"override,omitempty" lopt:"kerneloverride" sopt:"K" comment:"Set kernel override version" json:"override,omitempty"` @@ -118,3 +130,52 @@ type FileSystem struct { Options []string `yaml:"options,omitempty" comment:"any additional options to be passed to the format-specific mkfs utility"` MountOptions string `yaml:"mount_options,omitempty" comment:"any special options to be passed to the mount command"` } + +/* +interface so that nodes and profiles which aren't exported will +be marshaled +*/ +type ExportedYml struct { + WWInternal int `yaml:"WW_INTERNAL"` + NodeProfiles map[string]*ProfileConf `yaml:"nodeprofiles"` + Nodes map[string]*NodeConf `yaml:"nodes"` +} + +/* +Marshall Exported stuff, not NodeYaml directly +*/ +func (yml *NodeYaml) MarshalYAML() (interface{}, error) { + wwlog.Debug("marshall yml") + var exp ExportedYml + exp.WWInternal = yml.WWInternal + exp.Nodes = yml.nodes + exp.NodeProfiles = yml.nodeProfiles + node := yaml.Node{} + err := node.Encode(exp) + if err != nil { + return node, err + } + return node, err +} + +/* +Unmarshal to intermediate format +*/ +func (yml *NodeYaml) UnmarshalYAML( + unmarshal func(interface{}) (err error), +) (err error) { + wwlog.Debug("UnmarshalYAML called") + var exp ExportedYml + err = unmarshal(&exp) + if err != nil { + return + } + yml.WWInternal = exp.WWInternal + yml.nodes = exp.Nodes + yml.nodeProfiles = exp.NodeProfiles + return nil +} + +func (yml NodeYaml) IsZero() bool { + return true +} diff --git a/internal/pkg/node/flags.go b/internal/pkg/node/flags.go index 4b292f5c..aae6c332 100644 --- a/internal/pkg/node/flags.go +++ b/internal/pkg/node/flags.go @@ -5,6 +5,7 @@ import ( "reflect" "github.com/spf13/cobra" + "github.com/warewulf/warewulf/internal/pkg/wwtype" ) type NodeConfDel struct { @@ -71,6 +72,8 @@ func recursiveCreateFlags(obj interface{}, baseCmd *cobra.Command) { recursiveCreateFlags(nodeInfoVal.Elem().Field(i).MapIndex(key).Interface(), baseCmd) } else if nodeInfoType.Elem().Field(i).Anonymous { recursiveCreateFlags(nodeInfoVal.Elem().Field(i).Addr().Interface(), baseCmd) + } else if nodeInfoType.Elem().Field(i).Type.Kind() == reflect.Struct { + recursiveCreateFlags(nodeInfoVal.Elem().Field(i).Addr().Interface(), baseCmd) } } } @@ -182,6 +185,20 @@ func createFlags(baseCmd *cobra.Command, net.IPMask{}, // empty default! myType.Tag.Get("comment")) } + } else if myType.Type == reflect.TypeOf(wwtype.WWbool{}) { + ptr := myVal.Addr().Interface().(*wwtype.WWbool) + if myType.Tag.Get("sopt") != "" { + baseCmd.PersistentFlags().VarP(ptr, + myType.Tag.Get("lopt"), + myType.Tag.Get("sopt"), + myType.Tag.Get("comment")) + baseCmd.Flag(myType.Tag.Get("lopt")).NoOptDefVal = "true" + } else { + baseCmd.PersistentFlags().Var(ptr, + myType.Tag.Get("lopt"), + myType.Tag.Get("comment")) + baseCmd.Flag(myType.Tag.Get("lopt")).NoOptDefVal = "true" + } } } } diff --git a/internal/pkg/node/hash.go b/internal/pkg/node/hash.go index c309e1a0..1c57dce9 100644 --- a/internal/pkg/node/hash.go +++ b/internal/pkg/node/hash.go @@ -5,7 +5,7 @@ import ( "encoding/hex" "github.com/warewulf/warewulf/internal/pkg/wwlog" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) /* @@ -13,10 +13,10 @@ Calculate the hash of NodeYaml in an orderder fashion */ func (config *NodeYaml) Hash() [32]byte { // flatten out profiles and nodes - for _, val := range config.NodeProfiles { + for _, val := range config.nodeProfiles { val.Flatten() } - for _, val := range config.Nodes { + for _, val := range config.nodes { val.Flatten() } data, err := yaml.Marshal(config) diff --git a/internal/pkg/node/list.go b/internal/pkg/node/list.go index 1e92346a..b63c3cf8 100644 --- a/internal/pkg/node/list.go +++ b/internal/pkg/node/list.go @@ -1,7 +1,10 @@ package node import ( + "fmt" + "net" "reflect" + "strings" ) /* @@ -13,55 +16,91 @@ type NodeFields struct { Value string } +type fieldMap map[string]NodeFields + /* Get all the info out of NodeConf. If emptyFields is set true, all fields are shown not only the ones with effective values */ func (nodeYml *NodeYaml) GetFields(node NodeConf, emptyFields bool) (output []NodeFields) { - fieldMap := make(map[string]NodeFields) + nodeMap := make(fieldMap) for _, p := range node.Profiles { - if profile, ok := nodeYml.NodeProfiles[p]; ok { - recursiveFields(profile, emptyFields, "", fieldMap, p) + if profile, ok := nodeYml.nodeProfiles[p]; ok { + nodeMap.recursiveFields(profile, emptyFields, "", p) } } - recursiveFields(node, emptyFields, "", fieldMap, "") + nodeMap.recursiveFields(&node, emptyFields, "", "") + for _, elem := range nodeMap { + output = append(output, elem) + } + return output +} + +/* +Get all the info out of ProfileConf. If emptyFields is set true, all fields are shown not only the ones with effective values +*/ +func (nodeYml *NodeYaml) GetFieldsProfile(profile ProfileConf, emptyFields bool) (output []NodeFields) { + profileMap := make(fieldMap) + profileMap.recursiveFields(&profile, emptyFields, "", "") + for _, elem := range profileMap { + output = append(output, elem) + } return output } /* Internal function which travels through all fields of a NodeConf and for this -reason needs tb called via interface{} +reason needs to be called via interface{} */ -func recursiveFields(obj interface{}, emptyFields bool, prefix string, - fieldMap map[string]NodeFields, source string) { +func (fieldMap *fieldMap) recursiveFields(obj interface{}, emptyFields bool, prefix string, source string) { valObj := reflect.ValueOf(obj) typeObj := reflect.TypeOf(obj) for i := 0; i < typeObj.Elem().NumField(); i++ { + fmt.Printf("name: %s\n", typeObj.Elem().Field(i).Name) if valObj.Elem().Field(i).IsValid() { - if valObj.Elem().Field(i).String() != "" { - fieldMap[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + if !typeObj.Elem().Field(i).IsExported() { + continue + } + if valObj.Elem().Field(i).Kind() == reflect.String && valObj.Elem().Field(i).String() != "" { + fmt.Printf("string: %s\n", valObj.Elem().Field(i).String()) + (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ Field: prefix + typeObj.Elem().Field(i).Name, Source: source, Value: valObj.Elem().Field(i).String(), } } else if emptyFields { - fieldMap[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ Field: prefix + typeObj.Elem().Field(i).Name + "[]", Source: source, } - } - } else if typeObj.Elem().Field(i).Type.Kind() == reflect.Map { - mapIter := valObj.Elem().Field(i).MapRange() - for mapIter.Next() { - recursiveFields(mapIter.Value().Interface(), - emptyFields, prefix+typeObj.Elem().Field(i).Name+"["+mapIter.Key().String()+"].", fieldMap, source) - } - if valObj.Elem().Field(i).Len() == 0 && emptyFields { - fieldMap[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ - Field: prefix + typeObj.Elem().Field(i).Name + "[]", + } else if typeObj.Elem().Field(i).Type == reflect.TypeOf([]string{}) && valObj.Elem().Field(i).Len() != 0 { + vals := (valObj.Elem().Field(i).Interface()).([]string) + (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + Field: prefix + typeObj.Elem().Field(i).Name, + Source: source, + Value: strings.Join(vals, ","), } - } - } else if typeObj.Elem().Field(i).Type.Kind() == reflect.Ptr { - recursiveFields(valObj.Elem().Field(i).Interface(), emptyFields, prefix+typeObj.Elem().Field(i).Name+".", fieldMap, source) + } else if typeObj.Elem().Field(i).Type == reflect.TypeOf(net.IP{}) { + val := (valObj.Elem().Field(i).Interface()).(net.IP) + (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + Field: prefix + typeObj.Elem().Field(i).Name, + Source: source, + Value: val.String(), + } + } else if typeObj.Elem().Field(i).Type.Kind() == reflect.Map { + mapIter := valObj.Elem().Field(i).MapRange() + for mapIter.Next() { + fieldMap.recursiveFields(mapIter.Value().Interface(), + emptyFields, prefix+typeObj.Elem().Field(i).Name+"["+mapIter.Key().String()+"].", source) + } + if valObj.Elem().Field(i).Len() == 0 && emptyFields { + (*fieldMap)[prefix+typeObj.Elem().Field(i).Name] = NodeFields{ + Field: prefix + typeObj.Elem().Field(i).Name + "[]", + } + } + + } /*else if typeObj.Elem().Field(i).Type.Kind() == reflect.Ptr { + fieldMap.recursiveFields(valObj.Elem().Field(i).Interface(), emptyFields, prefix+typeObj.Elem().Field(i).Name+".", source) + }*/ } } } diff --git a/internal/pkg/node/methods.go b/internal/pkg/node/methods.go index 1a4eeb41..29e73eb6 100644 --- a/internal/pkg/node/methods.go +++ b/internal/pkg/node/methods.go @@ -1,6 +1,8 @@ package node import ( + "bytes" + "encoding/gob" "net" "reflect" "regexp" @@ -8,6 +10,7 @@ import ( "strings" "github.com/warewulf/warewulf/internal/pkg/util" + "github.com/warewulf/warewulf/internal/pkg/wwtype" ) type sortByName []NodeConf @@ -15,9 +18,6 @@ type sortByName []NodeConf func (a sortByName) Len() int { return len(a) } func (a sortByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a sortByName) Less(i, j int) bool { return a[i].id < a[j].id } -func GetUnsetVerbs() []string { - return []string{"UNSET", "DELETE", "UNDEF", "undef", "--", "nil", "0.0.0.0"} -} /********** * @@ -95,6 +95,15 @@ func NewNode(id string) (nodeconf NodeConf) { return nodeconf } +func EmptyNode() (nodeconf NodeConf) { + nodeconf.Ipmi = new(IpmiConf) + nodeconf.Ipmi.Tags = map[string]string{} + nodeconf.Kernel = new(KernelConf) + nodeconf.NetDevs = make(map[string]*NetDevs) + nodeconf.Tags = map[string]string{} + return nodeconf +} + /* Creates a ProfileConf but doesn't add it to the database. */ @@ -136,6 +145,7 @@ func recursiveFlatten(strct interface{}) { nestedType := reflect.TypeOf(confVal.Elem().Field(j).Interface()) nestedVal := reflect.ValueOf(confVal.Elem().Field(j).Interface()) for i := 0; i < nestedType.Elem().NumField(); i++ { + // wwlog.Debug("checking %s", nestedType.Elem().Field(i).Type.String()) if nestedType.Elem().Field(i).Type.Kind() == reflect.String && nestedVal.Elem().Field(i).Interface().(string) != "" && nestedVal.Elem().Field(i).Interface().(string) != undef { @@ -151,6 +161,11 @@ func recursiveFlatten(strct interface{}) { if len(val) != 0 && !val.IsUnspecified() { setToNil = false } + } else if nestedType.Elem().Field(i).Type == reflect.TypeOf(wwtype.WWbool{}) { + val := nestedVal.Elem().Field(i).Interface().(wwtype.WWbool) + if !val.IsZero() { + setToNil = false + } } } if setToNil { @@ -239,12 +254,19 @@ Getters for unexported fields */ /* -Returns the id of the node/profile +Returns the id of the node */ func (node *NodeConf) Id() string { return node.id } +/* +Returns the id of the profile +*/ +func (node *ProfileConf) Id() string { + return node.id +} + /* Returns if the node is a valid in the database */ @@ -258,3 +280,66 @@ Check if the netdev is the primary one func (dev *NetDevs) Primary() bool { return dev.primary } + +// returns all negated elements which are marked with ! as prefix +// from a list +func negList(list []string) (ret []string) { + for _, tok := range list { + if strings.HasPrefix(tok, "~") { + ret = append(ret, tok[1:]) + } + } + return +} + +// clean a list from negated tokens +func cleanList(list []string) (ret []string) { + neg := negList(list) + for _, listTok := range list { + notNegate := true + for _, negTok := range neg { + if listTok == negTok || listTok == "~"+negTok { + notNegate = false + } + } + if notNegate { + ret = append(ret, listTok) + } + } + return ret +} + +// Clone (deep copy) a node via gob +func (src *NodeConf) Clone() (dst NodeConf, err error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err = enc.Encode(src) + if err != nil { + return + } + err = dec.Decode(&dst) + if err != nil { + return + } + dst.id = src.id + dst.valid = src.valid + return +} + +// Clone (deep copy) a node via gob +func (src ProfileConf) Clone() (dst ProfileConf, err error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err = enc.Encode(src) + if err != nil { + return + } + err = dec.Decode(&dst) + if err != nil { + return + } + dst.id = src.id + return +} diff --git a/internal/pkg/node/modifiers.go b/internal/pkg/node/modifiers.go index 1f6b7d0c..a7f46736 100644 --- a/internal/pkg/node/modifiers.go +++ b/internal/pkg/node/modifiers.go @@ -1,10 +1,12 @@ package node import ( + "bytes" + "encoding/gob" "os" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/warewulf/warewulf/internal/pkg/wwlog" ) @@ -15,10 +17,10 @@ Add a node with the given ID and return a pointer to it func (config *NodeYaml) AddNode(nodeID string) (*NodeConf, error) { newNode := NewNode(nodeID) wwlog.Verbose("Adding new node: %s", nodeID) - if _, ok := config.Nodes[nodeID]; ok { + if _, ok := config.nodes[nodeID]; ok { return nil, errors.New("nodename already exists: " + nodeID) } else { - config.Nodes[nodeID] = &newNode + config.nodes[nodeID] = &newNode } return &newNode, nil } @@ -27,26 +29,64 @@ func (config *NodeYaml) AddNode(nodeID string) (*NodeConf, error) { delete node with the given id */ func (config *NodeYaml) DelNode(nodeID string) error { - if _, ok := config.Nodes[nodeID]; !ok { + if _, ok := config.nodes[nodeID]; !ok { return errors.New("nodename does not exist: " + nodeID) } wwlog.Verbose("Deleting node: %s", nodeID) - delete(config.Nodes, nodeID) + delete(config.nodes, nodeID) return nil } +/* +set node for the node with id the values of vals +*/ +func (config *NodeYaml) SetNode(nodeID string, vals NodeConf) error { + node, ok := config.nodes[nodeID] + if !ok { + return ErrNotFound + } + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err := enc.Encode(vals) + if err != nil { + return err + } + err = dec.Decode(node) + return err +} + +/* +set profile for the node with id the values of vals +*/ +func (config *NodeYaml) SetProfile(profileId string, vals ProfileConf) error { + profile, ok := config.nodeProfiles[profileId] + if !ok { + return ErrNotFound + } + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err := enc.Encode(vals) + if err != nil { + return err + } + err = dec.Decode(profile) + return err +} + /* Add a node with the given ID and return a pointer to it */ func (config *NodeYaml) AddProfile(profileId string) (*ProfileConf, error) { profile := NewProfile(profileId) wwlog.Verbose("adding new profile: %s", profileId) - if _, ok := config.NodeProfiles[profileId]; ok { + if _, ok := config.nodeProfiles[profileId]; ok { return nil, errors.New("profile already exists: " + profileId) } else { - config.NodeProfiles[profileId] = &profile + config.nodeProfiles[profileId] = &profile } return &profile, nil } @@ -55,12 +95,12 @@ func (config *NodeYaml) AddProfile(profileId string) (*ProfileConf, error) { delete node with the given id */ func (config *NodeYaml) DelProfile(nodeID string) error { - if _, ok := config.Nodes[nodeID]; !ok { + if _, ok := config.nodes[nodeID]; !ok { return errors.New("profile does not exist: " + nodeID) } wwlog.Verbose("deleting profile: %s", nodeID) - delete(config.Nodes, nodeID) + delete(config.nodes, nodeID) return nil } @@ -84,6 +124,7 @@ func (config *NodeYaml) Persist() error { if err != nil { return err } + wwlog.Debug("persisted: %s", ConfigFile) return nil } @@ -93,11 +134,16 @@ instance. Passes through any errors generated by yaml.Marshal. */ func (config *NodeYaml) Dump() ([]byte, error) { // flatten out profiles and nodes - for _, val := range config.NodeProfiles { + for _, val := range config.nodeProfiles { val.Flatten() } - for _, val := range config.Nodes { + for _, val := range config.nodes { val.Flatten() } - return yaml.Marshal(config) + var buf bytes.Buffer + // Run through encoder + yamlEncoder := yaml.NewEncoder(&buf) + yamlEncoder.SetIndent(2) + err := yamlEncoder.Encode(config) + return buf.Bytes(), err //yaml.Marshal(config) } diff --git a/internal/pkg/warewulfd/nodedb.go b/internal/pkg/warewulfd/nodedb.go index 7d5371ff..054b19ae 100644 --- a/internal/pkg/warewulfd/nodedb.go +++ b/internal/pkg/warewulfd/nodedb.go @@ -12,7 +12,7 @@ import ( type nodeDB struct { lock sync.RWMutex - NodeInfo map[string]node.NodeConf + NodeInfo map[string]string yml node.NodeYaml } @@ -21,13 +21,14 @@ var ( ) func LoadNodeDB() error { + db.lock.Lock() defer db.lock.Unlock() return loadNodeDB() } func loadNodeDB() (err error) { - TmpMap := make(map[string]node.NodeConf) + TmpMap := make(map[string]string) db.yml, err = node.New() if err != nil { @@ -45,7 +46,7 @@ func loadNodeDB() (err error) { } for _, netdev := range n.NetDevs { hwaddr := strings.ToLower(netdev.Hwaddr) - TmpMap[hwaddr] = n + TmpMap[hwaddr] = n.Id() } } @@ -53,82 +54,51 @@ func loadNodeDB() (err error) { return nil } -func GetNode(val string) (node node.NodeConf, err error) { +func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { db.lock.RLock() defer db.lock.RUnlock() - return getNode(val) -} - -func getNode(val string) (node node.NodeConf, err error) { - - if _, ok := db.NodeInfo[val]; ok { - - return db.NodeInfo[val], nil - } - - return node, errors.New("No node found") -} - -func GetNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { - db.lock.Lock() - defer db.lock.Unlock() - return getNodeOrSetDiscoverable(hwaddr) -} - -func PersistDb() error { - db.lock.Lock() - defer db.lock.Unlock() - return db.yml.Persist() -} - -func getNodeOrSetDiscoverable(hwaddr string) (node.NodeConf, error) { // NOTE: since discoverable nodes will write an updated DB to file and then // reload, it is not enough to lock individual reads from the DB // to ensure the condition on which the node is updated is still satisfied // after the DB is read back in. - n, err := getNode(hwaddr) - if err == nil { - return n, nil + nId, ok := db.NodeInfo[hwaddr] + if ok { + return db.yml.GetNode(nId) } // If we failed to find a node, let's see if we can add one... - var netdev string + wwlog.Warn("(node not configured)", hwaddr) - wwlog.WarnExc(err, "%s (node not configured)", hwaddr) - - nodeId, netdev, err := db.yml.FindDiscoverableNode() + node, netdev, err := db.yml.FindDiscoverableNode() if err != nil { // NOTE: this is taken as there is no discoverable node, so return the // empty one - return n, nil + return node, err } - discoverdNode, err := getNode(nodeId) + // update node + nodeChanges, _ := db.yml.GetNodeOnly(nId) // ignore error as nodeId is in db + wwlog.Debug("node: %v", nodeChanges) + nodeChanges.NetDevs[netdev].Hwaddr = hwaddr + nodeChanges.Discoverable = false + err = db.yml.SetNode(nId, nodeChanges) if err != nil { - return n, err + return node, err } - // update data on disk and in memory - discoverdNode.NetDevs[netdev].Hwaddr = hwaddr - db.yml.Nodes[discoverdNode.Id()].NetDevs[netdev].Hwaddr = hwaddr - discoverdNode.Discoverable = false - db.yml.Nodes[discoverdNode.Id()].Discoverable = false - - err = PersistDb() + err = db.yml.Persist() if err != nil { - return n, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr) + return node, errors.Wrapf(err, "%s (failed to persist node configuration)", hwaddr) } - err = loadNodeDB() if err != nil { - return n, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr) + return node, errors.Wrapf(err, "%s (failed to reload configuration)", hwaddr) } - // NOTE: previously all overlays were built here, but that will also // be done automatically when attempting to serve an overlay that // hasn't been built (without blocking the database). - wwlog.Serv("%s (node automatically configured)", hwaddr) + wwlog.Serv("%s (node %s automatically configured)", hwaddr, nId) // return the discovered node - return discoverdNode, nil + return db.yml.GetNode(nId) }