kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #894 from mslacken/FixProvisionBuild

Browse Source 
calling the overlay build functions direclty
This commit is contained in:
Jonathon Anderson
2023-10-25 22:40:15 -06:00
committed by GitHub
parent 92b651d70a 2b49bcad27
commit f1d45d2a0c
5 changed files with 344 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
internal/pkg/overlay/overlay.go
View File

@@ -19,25 +19,9 @@ import (
"github.com/pkg/errors"
)
/*
func BuildSystemOverlay(nodeList []node.NodeInfo) error {
return nil
}
func BuildRuntimeOverlay(nodeList []node.NodeInfo) error {
return nil
}
func FindSystemOverlays() ([]string, error) {
return findAllOverlays("system")
}
func FindRuntimeOverlays() ([]string, error) {
return findAllOverlays("runtime")
}
*/
var (
ErrDoesNotExist = errors.New("overlay does not exist")
)
/*
Build all overlays (runtime and generic) for a node
@@ -141,6 +125,9 @@ func OverlayInit(overlayName string) error {
Build the given overlays for a node and create a Image for them
*/
func BuildOverlay(nodeInfo node.NodeInfo, context string, overlayNames []string) error {
if len(overlayNames) == 0 {
return nil
}
// create the dir where the overlay images will reside
name := fmt.Sprintf("overlay %s/%v", nodeInfo.Id.Get(), overlayNames)
overlayImage := OverlayImage(nodeInfo.Id.Get(), context, overlayNames)
@@ -187,7 +174,7 @@ exists it will be created.
*/
func BuildOverlayIndir(nodeInfo node.NodeInfo, overlayNames []string, outputDir string) error {
if len(overlayNames) == 0 {
return errors.New("At least one valid overlay is needed to build for a node")
return nil
}
if !util.IsDir(outputDir) {
return errors.Errorf("output must a be a directory: %s", outputDir)
@@ -204,14 +191,10 @@ func BuildOverlayIndir(nodeInfo node.NodeInfo, overlayNames []string, outputDir
for _, overlayName := range overlayNames {
wwlog.Verbose("Building overlay %s for node %s in %s", overlayName, nodeInfo.Id.Get(), outputDir)
overlaySourceDir := OverlaySourceDir(overlayName)
wwlog.Debug("Starting to build overlay %s\nChanging directory to OverlayDir: %s", overlayName, overlaySourceDir)
wwlog.Debug("Changing directory to OverlayDir: %s", overlaySourceDir)
err := os.Chdir(overlaySourceDir)
if err != nil {
return errors.Wrap(err, "could not change directory to overlay dir")
}
wwlog.Debug("Checking to see if overlay directory exists: %s", overlaySourceDir)
if !util.IsDir(overlaySourceDir) {
return errors.New("overlay does not exist: " + overlayName)
return errors.Wrapf(ErrDoesNotExist, "directory: %s name: %s", overlaySourceDir, overlayName)
}
wwlog.Verbose("Walking the overlay structure: %s", overlaySourceDir)

290
internal/pkg/overlay/overlay_test.go
View File

@@ -9,7 +9,6 @@ import (
"os"
"path"
"sort"
"strings"
"testing"
)
@@ -20,19 +19,112 @@ var buildOverlayTests = []struct {
overlays []string
image string
contents []string
hasFiles bool
}{
{"empty", "", "", nil, "", nil},
{"empty node", "node1", "", nil, "", nil},
{"empty context", "", "system", nil, "", nil},
{"empty overlay", "", "", []string{"o1"}, "o1.img", []string{"o1.txt"}},
{"single overlay", "node1", "", []string{"o1"}, "node1/o1.img", []string{"o1.txt"}},
{"multiple overlays", "node1", "", []string{"o1", "o2"}, "node1/o1-o2.img", []string{"o1.txt", "o2.txt"}},
{"empty system overlay", "node1", "system", nil, "", nil},
{"empty runtime overlay", "node1", "runtime", nil, "", nil},
{"single system overlay", "node1", "system", []string{"o1"}, "node1/__SYSTEM__.img", []string{"o1.txt"}},
{"single runtime overlay", "node1", "runtime", []string{"o1"}, "node1/__RUNTIME__.img", []string{"o1.txt"}},
{"two system overlays", "node1", "system", []string{"o1", "o2"}, "node1/__SYSTEM__.img", []string{"o1.txt", "o2.txt"}},
{"two runtime overlays", "node1", "runtime", []string{"o1", "o2"}, "node1/__RUNTIME__.img", []string{"o1.txt", "o2.txt"}},
{
description: "if no node, context, or overlays are specified then no overlay image is generated",
nodeName: "",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only node is specified then no overlay image is generated",
nodeName: "node1",
context: "",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if only context is specified then no overlay image is generated",
nodeName: "",
context: "system",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if an overlay is specified without a node, then the overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1"},
image: "o1.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple overlays are specified without a node, then the combined overlay is built directly in the overlay directory",
nodeName: "",
context: "",
overlays: []string{"o1", "o2"},
image: "o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if a single node overlay is specified, then the overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1"},
image: "node1/o1.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple node overlays are specified, then the combined overlay is built in a node overlay directory",
nodeName: "node1",
context: "",
overlays: []string{"o1", "o2"},
image: "node1/o1-o2.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if no node system overlays are specified, then no overlay image is generated",
nodeName: "node1",
context: "system",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if no node runtime overlays are specified, then no overlay image is generated",
nodeName: "node1",
context: "runtime",
overlays: nil,
image: "",
contents: nil,
},
{
description: "if a single node system overlay is specified, then a system overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "system",
overlays: []string{"o1"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt"},
},
{
description: "if a single node runtime overlay is specified, then a runtime overlay image is generated in a node overlay directory",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt"},
},
{
description: "if multiple node system overlays are specified, then a system overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "system",
overlays: []string{"o1", "o2"},
image: "node1/__SYSTEM__.img",
contents: []string{"o1.txt", "o2.txt"},
},
{
description: "if multiple node runtime overlays are specified, then a runtime overlay image is generated with the contents of both overlays",
nodeName: "node1",
context: "runtime",
overlays: []string{"o1", "o2"},
image: "node1/__RUNTIME__.img",
contents: []string{"o1.txt", "o2.txt"},
},
}
func Test_BuildOverlay(t *testing.T) {
@@ -65,17 +157,16 @@ func Test_BuildOverlay(t *testing.T) {
conf.Paths.WWProvisiondir = provisionDir
err := BuildOverlay(nodeInfo, tt.context, tt.overlays)
if len(tt.image) > 0 {
assert.NoError(t, err)
if tt.image != "" {
image := path.Join(provisionDir, "overlays", tt.image)
assert.FileExists(t, image)
assert.NoError(t, err)
sort.Strings(tt.contents)
files := cpioFiles(t, image)
sort.Strings(files)
assert.Equal(t, tt.contents, files)
} else {
assert.Error(t, err)
dirName := path.Join(provisionDir, "overlays", tt.nodeName)
isEmpty := dirIsEmpty(t, dirName)
assert.True(t, isEmpty, "%v should be empty, but isn't", dirName)
@@ -84,28 +175,80 @@ func Test_BuildOverlay(t *testing.T) {
}
}
// Although these tests specify system and runtime overlays for the
// nodes, these overlays define the overlays that are defined in the
// configuration. BuildAllOverlays doesn't receive these as arguments,
// but builds all the overlays that are configured on the given node.
var buildAllOverlaysTests = []struct {
description string
nodes []string
systemOverlays []string
runtimeOverlays []string
succeed bool
systemOverlays [][]string
runtimeOverlays [][]string
createdOverlays []string
}{
{"no nodes", nil, nil, nil, true},
{"single empty node", []string{"node1"}, nil, nil, false},
{"two empty node", []string{"node1", "node2"}, nil, nil, false},
{"single node with system overlay", []string{"node1"},
[]string{"o1"}, nil, false},
{"two nodes with system overlays", []string{"node1", "node2"},
[]string{"o1", "o1,o2"}, nil, false},
{"single node with runtime overlay", []string{"node1"},
nil, []string{"o1"}, false},
{"two nodes with runtime overlays", []string{"node1", "node2"},
nil, []string{"o1", "o1,o2"}, false},
{"stingle node with full overlays", []string{"node1"},
[]string{"o1"}, []string{"o2"}, true},
{"two nodes with full overlays", []string{"node1", "node2"},
[]string{"o1", "o1,o2"}, []string{"o2", "o2"}, true},
{
description: "empty input creates no overlays",
nodes: nil,
systemOverlays: nil,
runtimeOverlays: nil,
createdOverlays: nil,
},
{
description: "a node with no overlays creates no overlays",
nodes: []string{"node1"},
systemOverlays: nil,
runtimeOverlays: nil,
createdOverlays: nil,
},
{
description: "multiple nodes with no overlays creates no overlays",
nodes: []string{"node1", "node2"},
systemOverlays: nil,
runtimeOverlays: nil,
createdOverlays: nil,
},
{
description: "a system overlay for a node generates a system overlay for that node",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with different system overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: nil,
createdOverlays: []string{"node1/__SYSTEM__.img.gz", "node2/__SYSTEM__.img.gz"},
},
{
description: "two nodes with a single runtime overlay generates a runtime overlay for the first node",
nodes: []string{"node1"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz"},
},
{
description: "two nodes with different runtime overlays generates a system overlay for each node",
nodes: []string{"node1", "node2"},
systemOverlays: nil,
runtimeOverlays: [][]string{{"o1"}, {"o1", "o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node2/__RUNTIME__.img.gz"},
},
{
description: "a node with both a runtime and system overlay generates an image for each",
nodes: []string{"node1"},
systemOverlays: [][]string{{"o1"}},
runtimeOverlays: [][]string{{"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz"},
},
{
description: "two nodes with both runtime and system overlays generates each image for each node",
nodes: []string{"node1", "node2"},
systemOverlays: [][]string{{"o1"}, {"o1", "o2"}},
runtimeOverlays: [][]string{{"o2"}, {"o2"}},
createdOverlays: []string{"node1/__RUNTIME__.img.gz", "node1/__SYSTEM__.img.gz", "node2/__RUNTIME__.img.gz", "node2/__SYSTEM__.img.gz"},
},
}
func Test_BuildAllOverlays(t *testing.T) {
@@ -129,22 +272,21 @@ func Test_BuildAllOverlays(t *testing.T) {
nodeInfo := node.NodeInfo{}
nodeInfo.Id.Set(nodeName)
if tt.systemOverlays != nil {
nodeInfo.SystemOverlay.SetSlice(strings.Split(tt.systemOverlays[i], ","))
nodeInfo.SystemOverlay.SetSlice(tt.systemOverlays[i])
}
if tt.runtimeOverlays != nil {
nodeInfo.RuntimeOverlay.SetSlice(strings.Split(tt.runtimeOverlays[i], ","))
nodeInfo.RuntimeOverlay.SetSlice(tt.runtimeOverlays[i])
}
nodes = append(nodes, nodeInfo)
}
err := BuildAllOverlays(nodes)
if !tt.succeed {
assert.Error(t, err)
} else {
assert.NoError(t, err)
for _, nodeName := range tt.nodes {
assert.FileExists(t, path.Join(provisionDir, "overlays", nodeName, "__SYSTEM__.img"))
assert.FileExists(t, path.Join(provisionDir, "overlays", nodeName, "__RUNTIME__.img"))
if tt.createdOverlays == nil {
dirName := path.Join(provisionDir, "overlays")
assert.True(t, dirIsEmpty(t, dirName), "%v should be empty, but isn't", dirName)
}
for _, overlayPath := range tt.createdOverlays {
assert.FileExists(t, path.Join(provisionDir, "overlays", overlayPath))
}
})
}
@@ -153,21 +295,59 @@ func Test_BuildAllOverlays(t *testing.T) {
var buildSpecificOverlaysTests = []struct {
description string
nodes []string
overlays string
overlays []string
images []string
succeed bool
}{
{"no nodes", nil, "", nil, true},
{"single empty node", []string{"node1"}, "", nil, false},
{"two empty node", []string{"node1", "node2"}, "", nil, false},
{"single node with single overlay", []string{"node1"}, "o1",
[]string{"node1/o1.img"}, true},
{"two nodes with single overlay", []string{"node1", "node2"}, "o1",
[]string{"node1/o1.img", "node2/o1.img"}, true},
{"single node with multi overlay", []string{"node1"}, "o1,o2",
[]string{"node1/o1.img", "node1/o2.img"}, true},
{"two nodes with multi overlays", []string{"node1", "node2"}, "o1,o2",
[]string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"}, true},
{
description: "building no overlays for no nodes generates no error and no images",
nodes: nil,
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for a node generates no error and no images",
nodes: []string{"node1"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building no overlays for two nodes generates no error and no images",
nodes: []string{"node1", "node2"},
overlays: nil,
images: nil,
succeed: true,
},
{
description: "building an overlay for a node generates an overlay image in that node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1"},
images: []string{"node1/o1.img"},
succeed: true,
},
{
description: "building an overlay for two nodes generates an overlay image in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1"},
images: []string{"node1/o1.img", "node2/o1.img"},
succeed: true,
},
{
description: "building multiple overlays for a node generates an overlay image for each overlay in the node's overlay directory",
nodes: []string{"node1"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img"},
succeed: true,
},
{
description: "building multiple overlays for two nodes generates an overlay image for each overlay in each node's overlay directory",
nodes: []string{"node1", "node2"},
overlays: []string{"o1", "o2"},
images: []string{"node1/o1.img", "node1/o2.img", "node2/o1.img", "node2/o2.img"},
succeed: true,
},
}
func Test_BuildSpecificOverlays(t *testing.T) {
@@ -192,7 +372,7 @@ func Test_BuildSpecificOverlays(t *testing.T) {
nodeInfo.Id.Set(nodeName)
nodes = append(nodes, nodeInfo)
}
err := BuildSpecificOverlays(nodes, strings.Split(tt.overlays, ","))
err := BuildSpecificOverlays(nodes, tt.overlays)
if !tt.succeed {
assert.Error(t, err)
} else {

10
internal/pkg/warewulfd/provision.go
View File

@@ -2,6 +2,7 @@ package warewulfd
import (
"bytes"
"errors"
"net/http"
"path"
"strconv"
@@ -11,6 +12,7 @@ import (
warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/container"
"github.com/hpcng/warewulf/internal/pkg/kernel"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
)
@@ -135,13 +137,19 @@ func ProvisionSend(w http.ResponseWriter, req *http.Request) {
} else {
context = rinfo.stage
}
stage_file, err = getOverlayFile(
node.Id.Get(),
node,
context,
request_overlays,
conf.Warewulf.AutobuildOverlays)
if err != nil {
if errors.Is(err, overlay.ErrDoesNotExist) {
w.WriteHeader(http.StatusNotFound)
wwlog.ErrorExc(err, "")
return
}
w.WriteHeader(http.StatusInternalServerError)
wwlog.ErrorExc(err, "")
return

23
internal/pkg/warewulfd/util.go
View File

@@ -4,6 +4,7 @@ import (
"net/http"
"os"
"github.com/hpcng/warewulf/internal/pkg/node"
nodepkg "github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/overlay"
"github.com/hpcng/warewulf/internal/pkg/util"
@@ -42,16 +43,15 @@ func sendFile(
}
func getOverlayFile(
nodeId string,
n node.NodeInfo,
context string,
stage_overlays []string,
autobuild bool) (stage_file string, err error) {
stage_file = overlay.OverlayImage(nodeId, context, stage_overlays)
stage_file = overlay.OverlayImage(n.Id.Get(), context, stage_overlays)
err = nil
build := !util.IsFile(stage_file)
wwlog.Verbose("stage file: %s", stage_file)
if !build && autobuild {
build = util.PathIsNewer(stage_file, nodepkg.ConfigFile)
@@ -61,21 +61,10 @@ func getOverlayFile(
}
if build {
wwlog.Serv("BUILD %15s, overlays %v", nodeId, stage_overlays)
args := []string{"overlay", "build"}
for _, overlayname := range stage_overlays {
args = append(args, "-O", overlayname)
}
args = append(args, nodeId)
out, err := util.RunWWCTL(args...)
err = overlay.BuildOverlay(n, context, stage_overlays)
if err != nil {
wwlog.Error("Failed to build overlay: %s, %s, %s\n%s",
nodeId, stage_overlays, stage_file, string(out))
n.Id.Get(), stage_overlays, stage_file, err)
}
}

99
internal/pkg/warewulfd/util_test.go
View File

@@ -1,9 +1,14 @@
package warewulfd
import (
warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
"github.com/stretchr/testify/assert"
"os"
"path"
"testing"
warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
"github.com/hpcng/warewulf/internal/pkg/node"
"github.com/hpcng/warewulf/internal/pkg/wwlog"
"github.com/stretchr/testify/assert"
)
var getOverlayFileTests = []struct {
@@ -12,30 +17,90 @@ var getOverlayFileTests = []struct {
context string
overlays []string
result string
succeed bool
}{
{"empty", "", "", nil, "", true},
{"empty node", "node1", "", nil, "", true},
{"specific overlays without node", "", "", []string{"o1", "o2"}, "p/overlays/o1-o2.img", true},
{"system overlay", "node1", "system", nil, "p/overlays/node1/__SYSTEM__.img", true},
{"runtime overlay", "node1", "runtime", nil, "p/overlays/node1/__RUNTIME__.img", true},
{"specific overlay", "node1", "", []string{"o1"}, "p/overlays/node1/o1.img", true},
{"multiple specific overlays", "node1", "", []string{"o1", "o2"}, "p/overlays/node1/o1-o2.img", true},
{"multiple specific overlays with context", "node1", "system", []string{"o1", "o2"}, "p/overlays/node1/__SYSTEM__.img", true},
{
description: "empty inputs produces no result",
node: "",
context: "",
overlays: nil,
result: "",
},
{
description: "a node with no context or overlays produces no result",
node: "node1",
context: "",
overlays: nil,
result: "",
},
{
description: "overlays with no node or context points to a combined overlay image",
node: "",
context: "",
overlays: []string{"o1", "o2"},
result: "overlays/o1-o2.img",
},
{
description: "system overlay for a node points to the node's system overlay image",
node: "node1",
context: "system",
overlays: []string{"o1"},
result: "overlays/node1/__SYSTEM__.img",
},
{
description: "runtime overlay for a node points to the node's runtime overlay image",
node: "node1",
context: "runtime",
overlays: nil,
result: "overlays/node1/__RUNTIME__.img",
},
{
description: "a specific overlay for a node points to that specific overlay image for that node",
node: "node1",
context: "",
overlays: []string{"o1"},
result: "overlays/node1/o1.img",
},
{
description: "a specific set of overlays for a node points to a combined overlay image for that node",
node: "node1",
context: "",
overlays: []string{"o1", "o2"},
result: "overlays/node1/o1-o2.img",
},
{
description: "a specific set of overlays for a node while also specifying a context points to the contextual overlay image for that node",
node: "node1",
context: "system",
overlays: []string{"o1", "o2"},
result: "overlays/node1/__SYSTEM__.img",
},
}
func Test_getOverlayFile(t *testing.T) {
wwlog.SetLogLevel(wwlog.DEBUG)
conf := warewulfconf.Get()
conf.Paths.WWProvisiondir = "p"
overlayPDir, overlayPDirErr := os.MkdirTemp(os.TempDir(), "ww-test-overlay-*")
assert.NoError(t, overlayPDirErr)
conf.Paths.WWProvisiondir = overlayPDir
overlayDir, overlayDirErr := os.MkdirTemp(os.TempDir(), "ww-test-provision-*")
assert.NoError(t, overlayDirErr)
conf.Paths.WWOverlaydir = overlayDir
defer os.RemoveAll(overlayDir)
assert.NoError(t, os.MkdirAll(path.Join(overlayDir, "o1"), 0700))
assert.NoError(t, os.WriteFile(path.Join(overlayDir, "o1", "test_file_o1"), []byte("test file"), 0600))
assert.NoError(t, os.MkdirAll(path.Join(overlayDir, "o2"), 0700))
for _, tt := range getOverlayFileTests {
t.Run(tt.description, func(t *testing.T) {
result, err := getOverlayFile(tt.node, tt.context, tt.overlays, false)
if !tt.succeed {
assert.Error(t, err)
} else {
var nodeInfo node.NodeInfo
nodeInfo.Id.Set(tt.node)
nodeInfo.RuntimeOverlay.SetSlice(tt.overlays)
nodeInfo.SystemOverlay.SetSlice(tt.overlays)
result, err := getOverlayFile(nodeInfo, tt.context, tt.overlays, false)
assert.NoError(t, err)
if tt.result != "" {
tt.result = path.Join(overlayPDir, tt.result)
}
assert.Equal(t, tt.result, result)
})
}