kelvin/warewulf
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Benchmarking for wwctl overlay build

Browse Source 
- Benchmarks overlay build in an environment with 1000 nodes
- Updates testenv to support both benchmarks and tests

To execute:

```
go test github.com/warewulf/warewulf/internal/app/wwctl/overlay/build -bench Benchmark_Overlay_Build
```

Signed-off-by: Jonathon Anderson <janderson@ciq.com>
This commit is contained in:
Jonathon Anderson
2024-12-23 18:17:57 -07:00
parent 276e67fc54
commit a18913924d
55 changed files with 3361 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
internal/pkg/testenv/testenv.go
View File

@@ -7,6 +7,7 @@ package testenv
import (
"os"
"os/exec"
"path"
"path/filepath"
"testing"
@@ -25,6 +26,8 @@ nodes:
`
type TestEnv struct {
t *testing.T
b *testing.B
BaseDir string
}
@@ -51,18 +54,42 @@ const Cachedir = "var/cache"
// Asserts no errors occur.
func New(t *testing.T) (env *TestEnv) {
env = new(TestEnv)
env.t = t
env.init()
return env
}
// NewBenchmark creates a benchmark environment in a temporary directory and configures
// Warewulf to use it.
//
// Caller is responsible to delete env.BaseDir by calling
// env.RemoveAll. Note that this does not restore Warewulf to its
// previous state.
//
// Asserts no errors occur.
func NewBenchmark(b *testing.B) (env *TestEnv) {
env = new(TestEnv)
env.b = b
env.init()
return env
}
// init creates a temporary directory and configured Warewulf to use it on behalf of
// New() or NewBenchmark().
//
// Asserts no errors occur.
func (env *TestEnv) init() {
tmpDir, err := os.MkdirTemp(os.TempDir(), "ww4test-*")
assert.NoError(t, err)
env.assertNoError(err)
env.BaseDir = tmpDir
env.WriteFile(t, path.Join(Sysconfdir, "warewulf/nodes.conf"), initNodesConf)
env.WriteFile(t, path.Join(Sysconfdir, "warewulf/warewulf.conf"), initWarewulfConf)
env.WriteFile(path.Join(Sysconfdir, "warewulf/nodes.conf"), initNodesConf)
env.WriteFile(path.Join(Sysconfdir, "warewulf/warewulf.conf"), initWarewulfConf)
// re-read warewulf.conf
conf := config.New()
err = conf.Read(env.GetPath(path.Join(Sysconfdir, "warewulf/warewulf.conf")))
assert.NoError(t, err)
env.assertNoError(err)
conf.Paths.Sysconfdir = env.GetPath(Sysconfdir)
conf.Paths.Bindir = env.GetPath(Bindir)
@@ -91,10 +118,24 @@ func New(t *testing.T) (env *TestEnv) {
conf.Paths.WWChrootdir,
conf.Paths.WWProvisiondir,
} {
env.MkdirAll(t, confPath)
env.MkdirAll(confPath)
}
}
// assertNoError handles error conditions generated by env,
// using the semantics of either a testing or a benchmark environment.
func (env *TestEnv) assertNoError(err error, msgAndArgs ...interface{}) {
if env.t != nil {
assert.NoError(env.t, err, msgAndArgs...)
}
return
if env.b != nil && err != nil {
if len(msgAndArgs) == 0 {
env.b.Errorf("%s", err)
} else {
env.b.Errorf(msgAndArgs[0].(string), (msgAndArgs[1:])...)
}
}
}
// GetPath returns the absolute path name for fileName specified
@@ -107,70 +148,77 @@ func (env *TestEnv) GetPath(fileName string) string {
// to the test environment.
//
// Asserts no errors occur.
func (env *TestEnv) MkdirAll(t *testing.T, dirName string) {
func (env *TestEnv) MkdirAll(dirName string) {
err := os.MkdirAll(env.GetPath(dirName), 0755)
assert.NoError(t, err)
env.assertNoError(err)
}
// Chmod changes the mode of fileName in the test environment, which must already exist.
//
// Asserts no errors occur.
func (env *TestEnv) Chmod(t *testing.T, fileName string, mode int) {
func (env *TestEnv) Chmod(fileName string, mode int) {
err := os.Chmod(env.GetPath(fileName), os.FileMode(mode))
assert.NoError(t, err)
env.assertNoError(err)
}
// WriteFile writes content to fileName, creating any necessary
// intermediate directories relative to the test environment.
//
// Asserts no errors occur.
func (env *TestEnv) WriteFile(t *testing.T, fileName string, content string) {
func (env *TestEnv) WriteFile(fileName string, content string) {
dirName := filepath.Dir(fileName)
env.MkdirAll(t, dirName)
env.MkdirAll(dirName)
f, err := os.Create(env.GetPath(fileName))
assert.NoError(t, err)
env.assertNoError(err)
defer f.Close()
_, err = f.WriteString(content)
assert.NoError(t, err)
env.assertNoError(err)
err = os.Chtimes(env.GetPath(fileName),
time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC),
time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC))
assert.NoError(t, err)
env.assertNoError(err)
}
// ImportFile writes the contents of inputFileName to fileName,
// creating any necessary intermediate directories relative to the
// test environment.
func (env *TestEnv) ImportFile(t *testing.T, fileName string, inputFileName string) {
func (env *TestEnv) ImportFile(fileName string, inputFileName string) {
buffer, err := os.ReadFile(inputFileName)
assert.NoError(t, err)
env.WriteFile(t, fileName, string(buffer))
env.assertNoError(err)
env.WriteFile(fileName, string(buffer))
}
func (env *TestEnv) ImportDir(dirName string, inputDirName string) {
env.MkdirAll(path.Dir(dirName))
cmd := exec.Command("cp", "--recursive", inputDirName, env.GetPath(dirName))
output, err := cmd.CombinedOutput()
env.assertNoError(err, string(output))
}
// CreateFile creates an empty file at fileName, creating any necessary intermediate directories
// relative to the test environment.
func (env *TestEnv) CreateFile(t *testing.T, fileName string) {
env.WriteFile(t, fileName, "")
func (env *TestEnv) CreateFile(fileName string) {
env.WriteFile(fileName, "")
}
// Symlink creates a symlink at fileName to target, creating any necessary intermediate directories
// relative to the test environment.
func (env *TestEnv) Symlink(t *testing.T, target string, fileName string) {
func (env *TestEnv) Symlink(target string, fileName string) {
dirName := filepath.Dir(fileName)
env.MkdirAll(t, dirName)
env.MkdirAll(dirName)
err := os.Symlink(target, env.GetPath(fileName))
assert.NoError(t, err)
env.assertNoError(err)
}
// ReadFile returns the content of fileName as converted to a
// string.
//
// Asserts no errors occur.
func (env *TestEnv) ReadFile(t *testing.T, fileName string) string {
func (env *TestEnv) ReadFile(fileName string) string {
buffer, err := os.ReadFile(env.GetPath(fileName))
assert.NoError(t, err)
env.assertNoError(err)
return string(buffer)
}
@@ -178,9 +226,9 @@ func (env *TestEnv) ReadFile(t *testing.T, fileName string) string {
// slice of strings.
//
// Asserts no errors occur.
func (env *TestEnv) ReadDir(t *testing.T, dirName string) []string {
func (env *TestEnv) ReadDir(dirName string) []string {
entries, err := os.ReadDir(env.GetPath(dirName))
assert.NoError(t, err)
env.assertNoError(err)
var entryStrs []string
for _, entry := range entries {
entryStrs = append(entryStrs, entry.Name())
@@ -192,7 +240,7 @@ func (env *TestEnv) ReadDir(t *testing.T, dirName string) []string {
// for the test environment.
//
// Asserts no errors occur.
func (env *TestEnv) RemoveAll(t *testing.T) {
func (env *TestEnv) RemoveAll() {
err := os.RemoveAll(env.BaseDir)
assert.NoError(t, err)
env.assertNoError(err)
}

6
internal/pkg/testenv/testenv_test.go
View File

@@ -9,7 +9,7 @@ import (
func Test_Basic(t *testing.T) {
env := New(t)
defer env.RemoveAll(t)
defer env.RemoveAll()
nodedb, err := node.New()
assert.NoError(t, err)
nodes, err := nodedb.FindAllNodes()
@@ -19,13 +19,13 @@ func Test_Basic(t *testing.T) {
func Test_two_nodes(t *testing.T) {
env := New(t)
env.WriteFile(t, "etc/warewulf/nodes.conf", `nodeprofiles:
env.WriteFile("etc/warewulf/nodes.conf", `nodeprofiles:
default: {}
nodes:
node1: {}
node2: {}
`)
defer env.RemoveAll(t)
defer env.RemoveAll()
nodedb, err := node.New()
assert.NoError(t, err)
nodes, err := nodedb.FindAllNodes()