From cac614a4b7ea684b5b4183dc47f1fb50c869fde9 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Sat, 10 Sep 2022 22:51:51 -0600 Subject: [PATCH 1/3] Correct handling of exclude patterns Signed-off-by: Jonathon Anderson --- internal/pkg/container/build.go | 13 +++---------- internal/pkg/util/util.go | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/internal/pkg/container/build.go b/internal/pkg/container/build.go index 147334ce..cd3e45d9 100644 --- a/internal/pkg/container/build.go +++ b/internal/pkg/container/build.go @@ -2,7 +2,6 @@ package container import ( "path" - "strings" "github.com/pkg/errors" @@ -27,20 +26,14 @@ func Build(name string, buildForce bool) error { } } - excludes_file := path.Join(rootfsPath, "./etc/warewulf/excludes") ignore := []string{} - + excludes_file := path.Join(rootfsPath, "./etc/warewulf/excludes") if util.IsFile(excludes_file) { - ignore, err := util.ReadFile(excludes_file) + var err error + ignore, err = util.ReadFile(excludes_file) if err != nil { return errors.Wrapf(err, "Failed creating directory: %s", imagePath) } - - for i, pattern := range ignore { - if ( strings.HasPrefix(pattern, "/") ) { - ignore[i] = pattern[1:] - } - } } err := util.BuildFsImage( diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 05ebfe0d..17b95a85 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -281,8 +281,9 @@ func FindFilterFiles( for i, pattern := range(ignore) { + ignore[i] = strings.TrimLeft(pattern, "/") if strings.HasPrefix(pattern, "./") { - ignore[i] = pattern[2:] + ignore[i] = strings.TrimPrefix(pattern, "./") } wwlog.Debug("Ignore pattern (%d): %s", i, ignore[i]) } @@ -309,6 +310,7 @@ func FindFilterFiles( // recursivly include from the matched directory num_init := len(ofiles) + ignoreDirs := []string{} err = filepath.Walk(ifile, func(location string, info os.FileInfo, err error) error { var file string @@ -331,6 +333,12 @@ func FindFilterFiles( return nil } + for _, ignoreDir := range(ignoreDirs) { + if strings.HasPrefix(location, ignoreDir) { + wwlog.Debug("Ignored (dir): %s", location) + return nil + } + } for i, pattern := range(ignore) { m, err := filepath.Match(pattern, location) if err != nil { @@ -339,6 +347,9 @@ func FindFilterFiles( if m { wwlog.Debug("Ignored (%d): %s", i, file) + if info.IsDir() { + ignoreDirs = append(ignoreDirs, file) + } return nil } } @@ -354,7 +365,7 @@ func FindFilterFiles( if err != nil { return ofiles, err } - }else{ + } else { wwlog.Debug("Included: %s", ifile) ofiles = append(ofiles, ifile) } From ff54fbc8d77f6da16bbc26451e94813f49e05390 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Sun, 11 Sep 2022 00:12:06 -0600 Subject: [PATCH 2/3] Simplify util.FindFilterFiles Signed-off-by: Jonathon Anderson --- internal/pkg/util/util.go | 148 ++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 85 deletions(-) diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 17b95a85..3db14a78 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -260,35 +260,17 @@ func FindFilterFiles( defer func() { err = FirstError(err, os.Chdir(cwd)) }() - err = os.Chdir(path) if err != nil { return ofiles, errors.Wrapf(err, "Failed to change path: %s", path) } - files := []string{} - - for _, pattern := range include { - - _files, err := filepath.Glob(pattern) - if err != nil { - return ofiles, errors.Wrapf(err, "Failed to apply pattern: %s", pattern) - } - wwlog.Debug("Including pattern: %s -> %d matches", pattern, len(_files)) - - files = append(files, _files...) - } - - - for i, pattern := range(ignore) { - ignore[i] = strings.TrimLeft(pattern, "/") - if strings.HasPrefix(pattern, "./") { - ignore[i] = strings.TrimPrefix(pattern, "./") - } + for i := range(ignore) { + ignore[i] = strings.TrimLeft(ignore[i], "/") + ignore[i] = strings.TrimPrefix(ignore[i], "./") wwlog.Debug("Ignore pattern (%d): %s", i, ignore[i]) } - if ignore_xdev { wwlog.Debug("Ignoring cross-device (xdev) files") } @@ -300,76 +282,72 @@ func FindFilterFiles( dev := path_stat.Sys().(*syscall.Stat_t).Dev - for _, ifile := range files { - stat, err := os.Stat(ifile) + + includeDirs := []string{} + ignoreDirs := []string{} + err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { if err != nil { - return ofiles, err + return err } - if stat.IsDir() { - // recursivly include from the matched directory + if location == "." { + return nil + } - num_init := len(ofiles) - ignoreDirs := []string{} - err = filepath.Walk(ifile, func(location string, info os.FileInfo, err error) error { - var file string - - if err != nil { - return err - } - - if location == "." { - return nil - } - - if info.IsDir() { - file = location + "/" - } else { - file = location - } - - if ignore_xdev && info.Sys().(*syscall.Stat_t).Dev != dev { - wwlog.Debug("Ignored (cross-device): %s", file) - return nil - } - - for _, ignoreDir := range(ignoreDirs) { - if strings.HasPrefix(location, ignoreDir) { - wwlog.Debug("Ignored (dir): %s", location) - return nil - } - } - for i, pattern := range(ignore) { - m, err := filepath.Match(pattern, location) - if err != nil { - return err - } - - if m { - wwlog.Debug("Ignored (%d): %s", i, file) - if info.IsDir() { - ignoreDirs = append(ignoreDirs, file) - } - return nil - } - } - - ofiles = append(ofiles, file) - - return nil - }) - - num_final := len(ofiles) - wwlog.Debug("Included: %s -> %d files", ifile, num_final-num_init) - - if err != nil { - return ofiles, err - } + var file string + if info.IsDir() { + file = location + "/" } else { - wwlog.Debug("Included: %s", ifile) - ofiles = append(ofiles, ifile) + file = location } - } + + if ignore_xdev && info.Sys().(*syscall.Stat_t).Dev != dev { + wwlog.Debug("Ignored (cross-device): %s", file) + return nil + } + + for _, ignoreDir := range(ignoreDirs) { + if strings.HasPrefix(location, ignoreDir) { + wwlog.Debug("Ignored (dir): %s", file) + return nil + } + } + for i, pattern := range(ignore) { + m, err := filepath.Match(pattern, location) + if err != nil { + return err + } else if m { + wwlog.Debug("Ignored (%d): %s", i, file) + if info.IsDir() { + ignoreDirs = append(ignoreDirs, file) + } + return nil + } + } + + for _, includeDir := range(includeDirs) { + if strings.HasPrefix(location, includeDir) { + wwlog.Debug("Included (dir): %s", file) + ofiles = append(ofiles, location) + return nil + } + } + for i, pattern := range(include) { + m, err := filepath.Match(pattern, location) + if err != nil { + return err + } else if m { + wwlog.Debug("Included (%d): %s", i, file) + ofiles = append(ofiles, location) + if info.IsDir() { + includeDirs = append(includeDirs, file) + } + return nil + } + } + + return nil + }) return ofiles, err } From 07a43ff164419083262cd31583561d9e32c03f74 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Sun, 11 Sep 2022 00:33:41 -0600 Subject: [PATCH 3/3] Update CHANGELOG for excludes fix Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index abc1ea9a..cb478793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - GID is no longer changed to `0` when unspecified during chown - Proper handling of uid/gid arguments during `wwctl overlay chown` +- /etc/warewulf/excludes stored in the container definition is now + processed correctly, also excluding the contents of an excluded + directory automatically. ## [4.1.0] - 2021-07-29