2026-04-29
This commit is contained in:
464
internal/pkg/util/util.go
Normal file
464
internal/pkg/util/util.go
Normal file
@@ -0,0 +1,464 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
//"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
//"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"gitea.sunhpc.com/kelvin/sunhpc/internal/pkg/splog"
|
||||
)
|
||||
|
||||
func IsDir(path string) bool {
|
||||
splog.Debug("Checking if path exists as a directory: %s", path)
|
||||
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
if stat, err := os.Stat(path); err == nil && stat.IsDir() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsFile(path string) bool {
|
||||
splog.Debug("Checking if path exists as a file: %s", path)
|
||||
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if stat, err := os.Lstat(path); err == nil && !stat.IsDir() {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ReadFile(path string) ([]string, error) {
|
||||
lines := []string{}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
_ = f.Close()
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
func ValidString(pattern string, s string) bool {
|
||||
if b, _ := regexp.MatchString(pattern, s); b {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func FindFiles(path string) []string {
|
||||
var ret []string
|
||||
|
||||
splog.Debug("Finding files in path: %s", path)
|
||||
|
||||
err := filepath.Walk(path, func(location string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
splog.Warn("Error walking path %s: %v", location, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get the relative path from the base directory
|
||||
relPath, relErr := filepath.Rel(path, location)
|
||||
if relErr != nil {
|
||||
splog.Warn("Error computing relative path for %s: %v", location, relErr)
|
||||
return relErr
|
||||
}
|
||||
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
splog.Debug("FindFiles() found directory: %s", relPath)
|
||||
ret = append(ret, relPath+"/")
|
||||
} else {
|
||||
splog.Debug("FindFiles() found file: %s", relPath)
|
||||
ret = append(ret, relPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
splog.Warn("Error during file walk: %v", err)
|
||||
return ret
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
/*
|
||||
Finds all files under a given directory with tar like include and ignore patterns.
|
||||
/foo/*
|
||||
will match /foo/baar/ and /foo/baar/sibling
|
||||
*/
|
||||
func FindFilterFiles(
|
||||
path string,
|
||||
includePattern []string,
|
||||
ignorePattern []string,
|
||||
ignore_xdev bool,
|
||||
) (ofiles []string, err error) {
|
||||
splog.Debug("Finding files: %s include: %s ignore: %s", path, includePattern, ignorePattern)
|
||||
|
||||
// Preprocess patterns to remove leading (and trailing) /, as we are handling relative paths
|
||||
for i, pattern := range ignorePattern {
|
||||
ignorePattern[i] = strings.Trim(pattern, "/")
|
||||
}
|
||||
|
||||
// Convert the base path to an absolute path
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return ofiles, fmt.Errorf("failed to resolve absolute path: %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Expand the include list
|
||||
var globedInclude []string
|
||||
for _, include := range includePattern {
|
||||
globed, err := filepath.Glob(filepath.Join(absPath, include))
|
||||
if err != nil {
|
||||
return ofiles, fmt.Errorf("failed to glob pattern %s: %w", include, err)
|
||||
}
|
||||
globedInclude = append(globedInclude, globed...)
|
||||
}
|
||||
|
||||
var dev uint64
|
||||
if ignore_xdev {
|
||||
splog.Debug("Ignoring cross-device (xdev) files")
|
||||
pathStat, err := os.Stat(absPath)
|
||||
if err != nil {
|
||||
return ofiles, fmt.Errorf("failed to stat base path: %s: %w", absPath, err)
|
||||
}
|
||||
dev = pathStat.Sys().(*syscall.Stat_t).Dev
|
||||
}
|
||||
|
||||
for _, inc := range globedInclude {
|
||||
splog.Debug("Processing include pattern: %s", inc)
|
||||
stat, err := os.Lstat(inc)
|
||||
if err != nil {
|
||||
return ofiles, fmt.Errorf("failed to stat include: %s: %w", inc, err)
|
||||
}
|
||||
|
||||
if stat.IsDir() {
|
||||
// Walk the directory
|
||||
err = filepath.WalkDir(inc, func(location string, info fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, relErr := filepath.Rel(absPath, location)
|
||||
if relErr != nil {
|
||||
splog.Warn("Error computing relative path for %s: %v", location, relErr)
|
||||
return relErr
|
||||
}
|
||||
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
fsInfo, err := info.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ignore_xdev && fsInfo.Sys().(*syscall.Stat_t).Dev != dev {
|
||||
splog.Debug("Ignored (cross-device): %s", location)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, ignoredPattern := range ignorePattern {
|
||||
if ignored, _ := filepath.Match(ignoredPattern, relPath); ignored {
|
||||
splog.Debug("Ignored %s due to pattern %s", relPath, ignoredPattern)
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ofiles = append(ofiles, relPath)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return ofiles, fmt.Errorf("error walking directory %s: %w", inc, err)
|
||||
}
|
||||
} else {
|
||||
// Add the file directly
|
||||
relPath, relErr := filepath.Rel(absPath, inc)
|
||||
if relErr != nil {
|
||||
splog.Warn("Error computing relative path for %s: %v", inc, relErr)
|
||||
return ofiles, relErr
|
||||
}
|
||||
ofiles = append(ofiles, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
return ofiles, nil
|
||||
}
|
||||
|
||||
// ******************************************************************************
|
||||
func ExecInteractive(command string, a ...string) error {
|
||||
splog.Debug("ExecInteractive(%s, %s)", command, a)
|
||||
c := exec.Command(command, a...)
|
||||
c.Stdin = os.Stdin
|
||||
c.Stdout = os.Stdout
|
||||
c.Stderr = os.Stderr
|
||||
err := c.Run()
|
||||
return err
|
||||
}
|
||||
|
||||
func SystemdStart(systemdName string) error {
|
||||
startCmd := fmt.Sprintf("systemctl restart %s", systemdName)
|
||||
enableCmd := fmt.Sprintf("systemctl enable %s", systemdName)
|
||||
|
||||
splog.Debug("Setting up Systemd service: %s", systemdName)
|
||||
err := ExecInteractive("/bin/sh", "-c", startCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run start cmd: %w", err)
|
||||
}
|
||||
err = ExecInteractive("/bin/sh", "-c", enableCmd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run enable cmd: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func CopyUIDGID(source string, dest string) error {
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// root is always good, if we failt to get UID/GID of a file
|
||||
var UID = 0
|
||||
var GID = 0
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
UID = int(stat.Uid)
|
||||
GID = int(stat.Gid)
|
||||
}
|
||||
splog.Debug("Chown %d:%d '%s'", UID, GID, dest)
|
||||
err = os.Chown(dest, UID, GID)
|
||||
return err
|
||||
}
|
||||
|
||||
func IncrementIPv4(start net.IP, inc uint) net.IP {
|
||||
ipv4 := start.To4()
|
||||
v4_int := uint(ipv4[0])<<24 + uint(ipv4[1])<<16 + uint(ipv4[2])<<8 + uint(ipv4[3])
|
||||
v4_int += inc
|
||||
v4_o3 := byte(v4_int & 0xFF)
|
||||
v4_o2 := byte((v4_int >> 8) & 0xFF)
|
||||
v4_o1 := byte((v4_int >> 16) & 0xFF)
|
||||
v4_o0 := byte((v4_int >> 24) & 0xFF)
|
||||
ipv4_new := net.IPv4(v4_o0, v4_o1, v4_o2, v4_o3)
|
||||
return ipv4_new
|
||||
}
|
||||
|
||||
/*
|
||||
Appending the lines to the given file
|
||||
*/
|
||||
func AppendLines(fileName string, lines []string) (err error) {
|
||||
splog.Verbose("appending %v lines to %s", len(lines), fileName)
|
||||
file, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't open file: %s: %w", fileName, err)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := file.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
for _, line := range lines {
|
||||
splog.Debug("Appending '%s' to %s", line, fileName)
|
||||
if _, err := fmt.Fprintf(file, "%s\n", line); err != nil {
|
||||
return fmt.Errorf("can't write to file: %s: %w", fileName, err)
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
******************************************************************************
|
||||
|
||||
Compress a file using gzip or pigz
|
||||
*/
|
||||
func FileGz(
|
||||
file string,
|
||||
) (err error) {
|
||||
file_gz := file + ".gz"
|
||||
|
||||
if IsFile(file_gz) {
|
||||
err := os.Remove(file_gz)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not remove existing file: %s: %w", file_gz, err)
|
||||
}
|
||||
}
|
||||
|
||||
compressor, err := exec.LookPath("pigz")
|
||||
if err != nil {
|
||||
splog.Verbose("Could not locate PIGZ")
|
||||
compressor, err = exec.LookPath("gzip")
|
||||
if err != nil {
|
||||
splog.Verbose("Could not locate GZIP")
|
||||
return fmt.Errorf("no compressor program for image file: %s: %w", file_gz, err)
|
||||
}
|
||||
}
|
||||
|
||||
splog.Verbose("Using compressor program: %s", compressor)
|
||||
|
||||
proc := exec.Command(
|
||||
compressor,
|
||||
"--keep",
|
||||
file)
|
||||
|
||||
out, err := proc.CombinedOutput()
|
||||
if len(out) > 0 {
|
||||
outStr := string(out[:])
|
||||
if err != nil && strings.HasSuffix(compressor, "gzip") && strings.Contains(outStr, "unrecognized option") {
|
||||
var gzippedFile *os.File
|
||||
var gzipStderr io.ReadCloser
|
||||
|
||||
/* Older version of gzip, try it another way: */
|
||||
splog.Verbose("%s does not recognize the --keep flag, trying redirected stdout", compressor)
|
||||
|
||||
/* Open the output file for writing: */
|
||||
gzippedFile, err = os.Create(file_gz)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open compressed image file for writing: %s: %w", file_gz, err)
|
||||
}
|
||||
|
||||
/* We'll execute gzip with output to stdout and attach stdout to the compressed file we just
|
||||
created:
|
||||
*/
|
||||
proc = exec.Command(
|
||||
compressor,
|
||||
"--stdout",
|
||||
file)
|
||||
proc.Stdout = gzippedFile
|
||||
gzipStderr, err = proc.StderrPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open stderr pipe for compression program: %s: %w", compressor, err)
|
||||
}
|
||||
|
||||
/* Execute the command: */
|
||||
err = proc.Start()
|
||||
if err != nil {
|
||||
_ = proc.Wait()
|
||||
_ = gzippedFile.Close()
|
||||
_ = os.Remove(file_gz)
|
||||
err = fmt.Errorf("unable to successfully execute compression program: %s: %w", compressor, err)
|
||||
} else {
|
||||
err = proc.Wait()
|
||||
if cerr := gzippedFile.Close(); cerr != nil {
|
||||
splog.Warn("failed to close compressed image file %s: %s", file_gz, cerr)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(file_gz)
|
||||
err = fmt.Errorf("unable to successfully create compressed image file: %s: %w", file_gz, err)
|
||||
} else {
|
||||
splog.Verbose("Successfully compressed image file: %s", file_gz)
|
||||
}
|
||||
}
|
||||
out, _ = io.ReadAll(gzipStderr)
|
||||
}
|
||||
splog.Debug(string(out))
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
/*
|
||||
Get size of given directory in bytes
|
||||
*/
|
||||
func DirSize(path string) (int64, error) {
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return err
|
||||
})
|
||||
return size, err
|
||||
}
|
||||
|
||||
/*
|
||||
Convert bytes to human friendly format
|
||||
*/
|
||||
func ByteToString(b int64) string {
|
||||
const base = 1024
|
||||
if b < base {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(base), 0
|
||||
for n := b / base; n >= base; n /= base {
|
||||
div *= base
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
func HashFile(file *os.File) (string, error) {
|
||||
if prevOffset, err := file.Seek(0, 0); err != nil {
|
||||
return "", err
|
||||
} else {
|
||||
hasher := sha256.New()
|
||||
if _, err := io.Copy(hasher, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := file.Seek(prevOffset, 0); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
}
|
||||
|
||||
func EncodeYaml(data interface{}) ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
encoder := yaml.NewEncoder(buf)
|
||||
encoder.SetIndent(2)
|
||||
err := encoder.Encode(data)
|
||||
return buf.Bytes(), err
|
||||
}
|
||||
|
||||
func EqualYaml(a interface{}, b interface{}) (bool, error) {
|
||||
aYaml, err := EncodeYaml(a)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
bYaml, err := EncodeYaml(b)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return bytes.Equal(aYaml, bYaml), nil
|
||||
}
|
||||
|
||||
func BoolP(p *bool) bool {
|
||||
return p != nil && *p
|
||||
}
|
||||
Reference in New Issue
Block a user