Reflection-based template walking

Rather than deriving the identity of template variables by string parsing the
name, identify its relationship to the underlying struct using reflection.

Signed-off-by: Jonathon Anderson <janderson@ciq.com>
This commit is contained in:
Jonathon Anderson
2025-12-04 00:26:03 -07:00
parent 29136e17d4
commit 4e67db0be6
5 changed files with 451 additions and 118 deletions

2
go.mod
View File

@@ -36,6 +36,7 @@ require (
github.com/swaggest/usecase v1.3.1
github.com/talos-systems/go-smbios v0.1.1
golang.org/x/crypto v0.32.0
golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67
golang.org/x/sys v0.29.0
golang.org/x/term v0.28.0
google.golang.org/genproto/googleapis/api v0.0.0-20250204164813-702378808489
@@ -149,7 +150,6 @@ require (
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/text v0.22.0 // indirect

View File

@@ -7,7 +7,6 @@ import (
"github.com/spf13/cobra"
"github.com/warewulf/warewulf/internal/app/wwctl/table"
"github.com/warewulf/warewulf/internal/pkg/node"
"github.com/warewulf/warewulf/internal/pkg/overlay"
"github.com/warewulf/warewulf/internal/pkg/wwlog"
"golang.org/x/exp/maps"
@@ -23,14 +22,14 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
return err
}
vars := ov.ParseVars(filePath)
sort.Strings(vars)
commentMap := ov.ParseCommentVars(filePath)
varMap := node.TemplateVarMap{}
varMap.ConfToTemplateMap(node.Node{}, "")
if vars == nil {
// Use new type-based variable resolution
varFields := ov.ParseVarFields(filePath)
if varFields == nil {
return fmt.Errorf("could not parse variables for %s in overlay %s", filePath, overlayName)
}
// Still parse comment vars for wwdoc and inline documentation
commentMap := ov.ParseCommentVars(filePath)
commentKeys := maps.Keys(commentMap)
sort.Strings(commentKeys)
for _, docLn := range commentKeys {
@@ -38,64 +37,56 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
wwlog.Info(commentMap[docLn])
}
}
// Sort variables by name for consistent output
varNames := maps.Keys(varFields)
sort.Strings(varNames)
t := table.New(cmd.OutOrStdout())
t.AddHeader("VARIABLE", "OPTION", "TYPE", "HELP")
for _, v := range vars {
found := false
helpText, hasCommentHelp := commentMap[v]
for _, varName := range varNames {
fieldInfo := varFields[varName]
helpText, hasCommentHelp := commentMap[varName]
for key, val := range varMap {
// fuzzy match, ignore case and try to also match singular / plural
textLower := strings.ToLower(v)
keyLower := strings.ToLower(key)
match := false
if strings.Contains(textLower, keyLower) {
match = true
} else {
keyParts := strings.Split(key, ".")
newParts := make([]string, len(keyParts))
for i, p := range keyParts {
newParts[i] = strings.ToLower(p)
}
for i, part := range newParts {
originalPart := part
var variation string
if strings.HasSuffix(part, "s") {
variation = strings.TrimSuffix(part, "s")
} else {
variation = part + "s"
}
newParts[i] = variation
variantKey := strings.Join(newParts, ".")
if strings.Contains(textLower, variantKey) {
match = true
break
}
newParts[i] = originalPart // restore for next iteration
}
// Extract metadata from the resolved field
opt := ""
typ := ""
help := ""
// Check if we have valid field information (field name is not empty)
hasValidField := fieldInfo.Field.Name != ""
if hasValidField {
// Get option from lopt tag
if lopt := fieldInfo.Field.Tag.Get("lopt"); lopt != "" {
opt = "--" + lopt
}
if match {
opt := ""
if val.LongOpt != "" {
opt = "--" + val.LongOpt
}
if hasCommentHelp {
t.AddLine(v, opt, val.Type, helpText)
} else {
t.AddLine(v, opt, val.Type, val.Comment)
}
found = true
// Get type from type tag, or use field type string
typ = fieldInfo.Field.Tag.Get("type")
if typ == "" {
// Use String() instead of Name() to handle composite types (slices, maps, pointers)
typ = fieldInfo.Field.Type.String()
}
// Get help from comment tag
help = fieldInfo.Field.Tag.Get("comment")
}
if !found {
if hasCommentHelp {
t.AddLine(v, "", "", helpText)
} else if strings.Contains(v, "Tags") {
t.AddLine(v, "", "", "", "")
}
// Prefer inline comment documentation if available
if hasCommentHelp {
help = helpText
}
// Special handling for Tags fields
if strings.Contains(varName, "Tags") {
t.AddLine(varName, opt, typ, help)
} else if hasValidField || hasCommentHelp {
t.AddLine(varName, opt, typ, help)
}
}
t.Print()
return nil
}

View File

@@ -324,32 +324,6 @@ type TemplateVarDetails struct {
LongOpt string
}
// Type to sore a map which looks and feels like the variables in a template
type TemplateVarMap map[string]TemplateVarDetails
// Fill the map so that every key is like a template variable and the value is the comment field
func (varMap TemplateVarMap) ConfToTemplateMap(obj interface{}, prefix string) {
objType := reflect.TypeOf(obj)
// now iterate of every field
for i := 0; i < objType.NumField(); i++ {
field := objType.Field(i)
if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct {
varMap.ConfToTemplateMap(reflect.New(field.Type.Elem()).Elem().Interface(), field.Name)
} else if field.Type.Kind() == reflect.Map && field.Type.Elem().Kind() == reflect.Ptr {
varMap.ConfToTemplateMap(reflect.New(field.Type.Elem().Elem()).Elem().Interface(), field.Name)
} else if field.Type.Kind() == reflect.Struct && field.Anonymous {
varMap.ConfToTemplateMap(reflect.New(field.Type).Elem().Interface(), field.Name)
} else {
varMap[prefix+"."+field.Name] = TemplateVarDetails{
Name: field.Name,
Comment: field.Tag.Get("comment"),
Type: field.Type.Name(),
LongOpt: field.Tag.Get("lopt"),
}
}
}
}
/*
Returns the id of the node
*/

View File

@@ -8,6 +8,7 @@ import (
"os"
"path"
"path/filepath"
"reflect"
"regexp"
"strings"
"sync"
@@ -273,8 +274,17 @@ func (overlay Overlay) Mkdir(path string, mode int32) (err error) {
return os.MkdirAll(fullPath, os.FileMode(mode))
}
// returns a list of the variable names for the given template
func (overlay Overlay) ParseVars(file string) []string {
// FieldInfo contains detailed type information about a template variable
type FieldInfo struct {
Field reflect.StructField // Complete field metadata including tags
ParentType reflect.Type // The containing struct type
FullPath string // Full path like ".NetDevs.Ipaddr6"
VarName string // Original variable name from template
}
// ParseVarFields returns detailed type information for each variable in the template
// by using reflection to resolve the actual struct fields being referenced.
func (overlay Overlay) ParseVarFields(file string) map[string]FieldInfo {
if !strings.HasSuffix(file, ".ww") {
return nil
}
@@ -291,17 +301,18 @@ func (overlay Overlay) ParseVars(file string) []string {
return nil
}
vars := make(map[string]bool)
result := make(map[string]FieldInfo)
rootType := reflect.TypeOf(TemplateStruct{})
// Track range variables and their types
rangeVars := make(map[string]reflect.Type)
// Initialize $ to refer to the root template context
rangeVars["$"] = rootType
if tmpl.Tree != nil && tmpl.Tree.Root != nil {
walkParseTree(tmpl.Tree.Root, vars)
walkParseTree(tmpl.Tree.Root, rootType, "", rangeVars, result)
}
result := make([]string, 0, len(vars))
for v := range vars {
if v != "" {
result = append(result, v)
}
}
return result
}
@@ -339,47 +350,402 @@ func (overlay Overlay) ParseCommentVars(file string) (retMap map[string]string)
return
}
// walkParseTree recursively traverses the template's parse tree to find variables.
func walkParseTree(node parse.Node, vars map[string]bool) {
// walkParseTree recursively traverses the template's parse tree and resolves
// variable references to actual struct fields using reflection.
func walkParseTree(node parse.Node, currentType reflect.Type, currentPath string, rangeVars map[string]reflect.Type, result map[string]FieldInfo) {
if node == nil {
return
}
switch n := node.(type) {
case *parse.ActionNode:
walkParseTree(n.Pipe, vars)
// Handle variable assignments like $var := expr
if n.Pipe != nil && len(n.Pipe.Decl) > 0 && len(n.Pipe.Cmds) > 0 {
// Try to resolve the type of the expression being assigned
cmd := n.Pipe.Cmds[0]
if len(cmd.Args) > 0 {
// Check for field access (e.g., $NetDevs := .NetDevs)
if field, ok := cmd.Args[0].(*parse.FieldNode); ok {
fieldInfo := resolveFieldChain(currentType, field.Ident, currentPath)
if fieldInfo != nil {
// Record all declared variables with this type
for _, decl := range n.Pipe.Decl {
rangeVars[decl.Ident[0]] = fieldInfo.Field.Type
}
}
} else if varNode, ok := cmd.Args[0].(*parse.VariableNode); ok {
// Check for variable-to-variable assignment (e.g., $b := $a)
if len(varNode.Ident) == 1 {
// Simple variable reference
varBaseName := varNode.Ident[0]
if varType, exists := rangeVars[varBaseName]; exists {
for _, decl := range n.Pipe.Decl {
rangeVars[decl.Ident[0]] = varType
}
}
}
}
}
}
walkParseTree(n.Pipe, currentType, currentPath, rangeVars, result)
case *parse.IfNode:
walkParseTree(n.Pipe, vars)
walkParseTree(n.List, vars)
walkParseTree(n.ElseList, vars)
walkParseTree(n.Pipe, currentType, currentPath, rangeVars, result)
walkParseTree(n.List, currentType, currentPath, rangeVars, result)
walkParseTree(n.ElseList, currentType, currentPath, rangeVars, result)
case *parse.ListNode:
if n != nil {
for _, child := range n.Nodes {
walkParseTree(child, vars)
walkParseTree(child, currentType, currentPath, rangeVars, result)
}
}
case *parse.RangeNode:
walkParseTree(n.Pipe, vars)
walkParseTree(n.List, vars)
walkParseTree(n.ElseList, vars)
// Handle range statements like {{range $key, $val := .NetDevs}}
// First, walk the pipe to record the variable being ranged over
walkParseTree(n.Pipe, currentType, currentPath, rangeVars, result)
if n.Pipe != nil && len(n.Pipe.Cmds) > 0 {
// Get what's being ranged over to determine element type
var fieldInfo *FieldInfo
// Check for both FieldNode (e.g., .NetDevs) and VariableNode (e.g., $disk.PartitionList)
rangeField := extractFieldFromPipe(n.Pipe)
if rangeField != nil {
// Resolve the type of the field being ranged over
fieldInfo = resolveFieldChain(currentType, rangeField.Ident, currentPath)
} else {
// Check if it's a variable access
for _, cmd := range n.Pipe.Cmds {
for _, arg := range cmd.Args {
if varNode, ok := arg.(*parse.VariableNode); ok {
varBaseName := varNode.Ident[0]
if varType, exists := rangeVars[varBaseName]; exists {
if len(varNode.Ident) > 1 {
// Multi-part variable like $disk.PartitionList
fieldIdents := varNode.Ident[1:] // Skip the variable name
fieldInfo = resolveFieldChain(varType, fieldIdents, currentPath)
} else {
// Simple variable like $NetDevs
fieldInfo = &FieldInfo{
Field: reflect.StructField{
Name: varBaseName,
Type: varType,
},
ParentType: currentType,
FullPath: currentPath,
}
}
break
}
}
}
if fieldInfo != nil {
break
}
}
}
if fieldInfo != nil {
rangeType := fieldInfo.Field.Type
// For slices and arrays, get the element type
if rangeType.Kind() == reflect.Slice || rangeType.Kind() == reflect.Array {
rangeType = rangeType.Elem()
}
// For maps, get the value type
if rangeType.Kind() == reflect.Map {
rangeType = rangeType.Elem()
}
// Dereference pointers
if rangeType.Kind() == reflect.Ptr {
rangeType = rangeType.Elem()
}
// Track the range variable assignments
if len(n.Pipe.Decl) > 0 {
if len(n.Pipe.Decl) == 2 {
// Two-variable range: $key, $val := .Map or $index, $val := .Slice
keyType := reflect.TypeOf(0) // Default to int for slice indices
if fieldInfo.Field.Type.Kind() == reflect.Map {
keyType = fieldInfo.Field.Type.Key()
}
rangeVars[n.Pipe.Decl[0].Ident[0]] = keyType // First variable is key/index
rangeVars[n.Pipe.Decl[1].Ident[0]] = rangeType // Second variable is value
} else {
// Single-variable range: $val := .Slice
rangeVars[n.Pipe.Decl[0].Ident[0]] = rangeType
}
}
// Walk the range body with the element type
walkParseTree(n.List, rangeType, fieldInfo.FullPath, rangeVars, result)
}
}
walkParseTree(n.ElseList, currentType, currentPath, rangeVars, result)
case *parse.WithNode:
walkParseTree(n.Pipe, vars)
walkParseTree(n.List, vars)
walkParseTree(n.ElseList, vars)
walkParseTree(n.Pipe, currentType, currentPath, rangeVars, result)
walkParseTree(n.List, currentType, currentPath, rangeVars, result)
walkParseTree(n.ElseList, currentType, currentPath, rangeVars, result)
case *parse.TemplateNode:
walkParseTree(n.Pipe, vars)
case *parse.BreakNode, *parse.ContinueNode:
// No variables to extract
walkParseTree(n.Pipe, currentType, currentPath, rangeVars, result)
case *parse.PipeNode:
if n != nil {
for _, cmd := range n.Cmds {
for _, arg := range cmd.Args {
walkParseTree(arg, vars)
walkParseTree(arg, currentType, currentPath, rangeVars, result)
}
}
}
case *parse.VariableNode, *parse.FieldNode:
vars[n.String()] = true
case *parse.FieldNode:
// Field access like .Ipmi.Ipaddr or $netdev.Ipaddr
varName := n.String()
// Determine the base type for field resolution
baseType := currentType
basePath := currentPath
// Check if this is a variable access (starts with $)
if strings.HasPrefix(varName, "$") {
// Extract variable name (e.g., "$netdev.Device" -> "netdev")
parts := strings.SplitN(varName[1:], ".", 2)
if len(parts) > 0 {
varBaseName := parts[0]
if varType, exists := rangeVars[varBaseName]; exists {
baseType = varType
basePath = currentPath
}
}
}
fieldInfo := resolveFieldChain(baseType, n.Ident, basePath)
// If resolution failed, try adding "P" suffix to last identifier
// This handles methods like Primary() backed by PrimaryP field
if fieldInfo == nil && len(n.Ident) >= 1 {
identWithP := make([]string, len(n.Ident))
copy(identWithP, n.Ident)
identWithP[len(identWithP)-1] += "P"
fieldInfo = resolveFieldChain(baseType, identWithP, basePath)
// Use the original variable name (without P)
if fieldInfo != nil {
fieldInfo.VarName = varName
}
}
if fieldInfo != nil {
if fieldInfo.VarName == "" {
fieldInfo.VarName = varName
}
result[varName] = *fieldInfo
}
case *parse.VariableNode:
// Variable reference like $netdev or possibly $netdev.Field
varName := n.String()
// Check if this is a simple variable or a field access on a variable
if len(n.Ident) > 1 {
// This is $var.Field or $var.Field.Method - handle like a FieldNode
varBaseName := n.Ident[0]
if varType, exists := rangeVars[varBaseName]; exists {
// Resolve the field chain starting from the variable's type
fieldIdents := n.Ident[1:] // Skip the variable name, keep the fields
fieldInfo := resolveFieldChain(varType, fieldIdents, currentPath)
// If resolution failed and we have multiple parts, the last part might be a method
// Try resolving without the last identifier (e.g., OnBoot.BoolDefaultTrue -> OnBoot)
if fieldInfo == nil && len(fieldIdents) > 1 {
fieldIdents = fieldIdents[:len(fieldIdents)-1]
fieldInfo = resolveFieldChain(varType, fieldIdents, currentPath)
// Use a simplified variable name without the method
if fieldInfo != nil {
// Build simplified name: $varBaseName.field1.field2 (without method)
simplifiedName := varBaseName
for _, ident := range fieldIdents {
simplifiedName += "." + ident
}
fieldInfo.VarName = simplifiedName
}
}
// If resolution still failed, try adding "P" suffix to last identifier
// This handles methods like Primary() backed by PrimaryP field
if fieldInfo == nil && len(fieldIdents) >= 1 {
// Try with "P" suffix on the last identifier
fieldIdentsWithP := make([]string, len(fieldIdents))
copy(fieldIdentsWithP, fieldIdents)
fieldIdentsWithP[len(fieldIdentsWithP)-1] += "P"
fieldInfo = resolveFieldChain(varType, fieldIdentsWithP, currentPath)
// Use the original variable name (without P)
if fieldInfo != nil {
fieldInfo.VarName = varName
}
}
if fieldInfo != nil {
if fieldInfo.VarName == "" {
fieldInfo.VarName = varName
}
result[fieldInfo.VarName] = *fieldInfo
}
}
} else if len(n.Ident) == 1 {
// Simple variable reference
varBaseName := n.Ident[0]
if varType, exists := rangeVars[varBaseName]; exists {
result[varName] = FieldInfo{
VarName: varName,
ParentType: varType,
FullPath: currentPath,
}
}
}
}
}
// extractFieldFromPipe extracts the FieldNode from a pipe (used in range statements)
func extractFieldFromPipe(pipe *parse.PipeNode) *parse.FieldNode {
if pipe == nil || len(pipe.Cmds) == 0 {
return nil
}
for _, cmd := range pipe.Cmds {
for _, arg := range cmd.Args {
if field, ok := arg.(*parse.FieldNode); ok {
return field
}
}
}
return nil
}
// resolveFieldChain walks a chain of field identifiers (like ["Ipmi", "Ipaddr"])
// and returns the final field's information using reflection.
// Methods are resolved and reported. If a method has a backing field with "P" suffix,
// the backing field's metadata is used; otherwise, the method's return type is used.
func resolveFieldChain(rootType reflect.Type, idents []string, basePath string) *FieldInfo {
if rootType.Kind() == reflect.Ptr {
rootType = rootType.Elem()
}
if len(idents) == 0 {
return nil
}
currentType := rootType
fullPath := basePath
var finalField reflect.StructField
var parentType reflect.Type
for i, fieldName := range idents {
if currentType.Kind() != reflect.Struct {
return nil
}
field, found := currentType.FieldByName(fieldName)
if !found {
// Field not found - try to find a method with this name
// This handles methods like DiskList(), PartitionList(), Id(), ShouldExist()
ptrType := reflect.PointerTo(currentType)
method, methodFound := ptrType.MethodByName(fieldName)
if !methodFound {
return nil
}
// Get the method's return type (first return value)
methodType := method.Type
if methodType.NumOut() == 0 {
return nil
}
returnType := methodType.Out(0)
// Check if there's a backing field with "P" suffix
// Only methods with backing fields should be reported as user-facing variables
backingFieldName := fieldName + "P"
backingField, backingFound := currentType.FieldByName(backingFieldName)
// For the last identifier in the chain
if i == len(idents)-1 {
if backingFound {
// Use the backing field's metadata (tags) but the method's return type
// This gives us the documentation from ShouldExistP but the type from ShouldExist()
finalField = reflect.StructField{
Name: fieldName,
Type: returnType, // Use method's return type, not backing field's type
Tag: backingField.Tag,
}
parentType = currentType
fullPath += "." + fieldName // Use method name in path, not field name
currentType = returnType
} else {
// No backing field - create a field with the method's return type
// This allows documenting methods via comments
finalField = reflect.StructField{
Name: fieldName,
Type: returnType,
}
parentType = currentType
fullPath += "." + fieldName
currentType = returnType
}
} else {
// Not the last identifier - we're in the middle of a chain (e.g., DiskList in node.DiskList.PartitionList)
// Continue walking with the method's return type for type resolution
parentType = currentType
fullPath += "." + fieldName
currentType = returnType
// We don't have a real field yet, just continue to next identifier
// Don't set finalField here as we need to keep walking
continue
}
} else {
// Field found
finalField = field
parentType = currentType
fullPath += "." + fieldName
currentType = field.Type
}
// Dereference pointer types for next iteration
if currentType.Kind() == reflect.Ptr {
currentType = currentType.Elem()
}
// For map types, remaining identifiers are map keys, not fields
if currentType.Kind() == reflect.Map && i < len(idents)-1 {
// Append remaining path as map keys
mapKeyPath := ""
for j := i + 1; j < len(idents); j++ {
mapKeyPath += "." + idents[j]
}
fullPath += mapKeyPath
// Create a synthetic field with the map's value type
// For Tags (map[string]string), accessing Tags.key should return string, not map[string]string
valueType := currentType.Elem()
return &FieldInfo{
Field: reflect.StructField{
Name: idents[len(idents)-1], // Use the last key as the field name
Type: valueType, // Use the map's value type
},
ParentType: parentType,
FullPath: fullPath,
}
}
}
return &FieldInfo{
Field: finalField,
ParentType: parentType,
FullPath: fullPath,
}
}

View File

@@ -2,10 +2,10 @@
network:
version: 2
renderer: networkd
{{- $NetDevs := .NetDevs }}
{{- $ethernets := list }}
{{- $bonds := list }}
{{- $NetDevs := .NetDevs }}
{{- range $devname, $netdev := .NetDevs }}
{{- range $devname, $netdev := $NetDevs }}
{{- if $netdev.Device }}
{{- if eq (default "ethernet" (lower $netdev.Type)) "ethernet" }}
{{- $ethernets = append $ethernets $devname }}
@@ -16,8 +16,8 @@ network:
{{- end }}
{{- if $ethernets }}
ethernets:
{{- range $devname := $ethernets }}
{{- $netdev := (index $NetDevs $devname) }}
{{- range $netdev := $NetDevs }}
{{- if eq (default "ethernet" (lower $netdev.Type)) "ethernet" }}
{{$netdev.Device}}:
dhcp4: no
optional: true
@@ -30,10 +30,11 @@ network:
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if $bonds }}
bonds:
{{- range $devname := $bonds }}
{{- $netdev := (index $NetDevs $devname) }}
{{- range $netdev := $NetDevs }}
{{- if eq (lower $netdev.Type) "bond" }}
{{$netdev.Device}}:
dhcp4: no
optional: true
@@ -61,3 +62,4 @@ network:
{{- end }}
{{- end }}
{{- end }}
{{- end }}