diff --git a/Defaults.mk.in b/Defaults.mk.in deleted file mode 100644 index a16e4e53..00000000 --- a/Defaults.mk.in +++ /dev/null @@ -1,14 +0,0 @@ -VERSION ?= @VERSION@ -RELEASE ?= @RELEASE@ -PREFIX ?= @PREFIX@ -BINDIR ?= @BINDIR@ -SYSCONFDIR ?= @SYSCONFDIR@ -SRVDIR ?= @SRVDIR@ -DATADIR ?= @DATADIR@ -MANDIR ?= @MANDIR@ -LOCALSTATEDIR ?= @LOCALSTATEDIR@ -TFTPDIR ?= @TFTPDIR@ -FIREWALLDDIR ?= @FIREWALLDDIR@ -SYSTEMDDIR ?= @SYSTEMDDIR@ -BASH_COMPLETION ?= @BASH_COMPLETION@ -WWCLIENTLOC ?= @WWCLIENTLOC@ diff --git a/Makefile b/Makefile index 72b98588..7bf11b7f 100644 --- a/Makefile +++ b/Makefile @@ -2,38 +2,80 @@ -include Defaults.mk -VERSION ?= 4.2.0 -GIT_TAG := $(shell test -e .git && git describe --tags --long --first-parent --always | tr '-' '_') +# Linux distro (try and set to /etc/os-release ID) +OS_REL := $(shell sed -n "s/^ID\s*=\s*['"\""]\(.*\)['"\""]/\1/p" /etc/os-release) +OS ?= $(OS_REL) -ifndef $(GIT_TAG) - RELEASE ?= 1.git_$(GIT_TAG) +# List of variables to save and replace in files +VARLIST := OS + +# Project Information +VARLIST += WAREWULF VERSION RELEASE +WAREWULF ?= warewulf +VERSION ?= 4.2.0 +GIT_TAG := $(shell test -e .git && git describe --tags --long --first-parent --always) + +ifndef GIT_TAG + ifdef $(filter $(OS),ubuntu debian) + RELEASE ?= 1.git_$(subst -,+,$(GIT_TAG)) + else + RELEASE ?= 1.git_$(subst -,_,$(GIT_TAG)) + endif else - RELEASE ?= 1 + RELEASE ?= 1 endif -# System locations +# Use LSB-compliant paths if OS is known +ifneq ($(OS),) + USE_LSB_PATHS := true +endif + +# Always default to GNU autotools default paths if PREFIX has been redefined +ifdef PREFIX + USE_LSB_PATHS := false +endif + +# System directory paths +VARLIST += PREFIX BINDIR SYSCONFDIR SRVDIR DATADIR MANDIR DOCDIR LOCALSTATEDIR PREFIX ?= /usr/local BINDIR ?= $(PREFIX)/bin SYSCONFDIR ?= $(PREFIX)/etc -SRVDIR ?= $(PREFIX)/srv DATADIR ?= $(PREFIX)/share MANDIR ?= $(DATADIR)/man -LOCALSTATEDIR ?= $(PREFIX)/var -WWCLIENTLOC ?= /warewulf/bin +DOCDIR ?= $(DATADIR)/doc -TFTPDIR ?= /var/lib/tftpboot -FIREWALLDDIR ?= /usr/lib/firewalld/services +ifeq ($(USE_LSB_PATHS),true) + SRVDIR ?= /srv + LOCALSTATEDIR ?= /var/local +else + SRVDIR ?= $(PREFIX)/srv + LOCALSTATEDIR ?= $(PREFIX)/var +endif + +# OS-Specific Service Locations +VARLIST += TFTPDIR FIREWALLDDIR SYSTEMDDIR SYSTEMDDIR ?= /usr/lib/systemd/system -BASH_COMPLETION ?= /etc/bash_completion.d/ +BASHCOMPDIR ?= /etc/bash_completion.d +FIREWALLDDIR ?= /usr/lib/firewalld/services +ifeq ($(OS),suse) + TFTPDIR ?= /srv/tftpboot +endif +ifeq ($(OS),ubuntu) + TFTPDIR ?= /srv/tftp +endif +# Default to Red Hat / Rocky Linux +TFTPDIR ?= /var/lib/tftpboot -# Warewulf locations -WWPROVISIONDIR = $(SRVDIR)/warewulf -WWOVERLAYDIR = $(LOCALSTATEDIR)/warewulf/overlays -WWCHROOTDIR = $(LOCALSTATEDIR)/warewulf/chroots - -# SuSE -#TFTPDIR ?= /srv/tftpboot -#FIREWALLDIR ?= /srv/tftp +# Warewulf directory paths +VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR WWDATADIR +WWCONFIGDIR := $(SYSCONFDIR)/$(WAREWULF) +WWPROVISIONDIR := $(LOCALSTATEDIR)/$(WAREWULF) +WWOVERLAYDIR := $(LOCALSTATEDIR)/$(WAREWULF)/overlays +WWCHROOTDIR := $(LOCALSTATEDIR)/$(WAREWULF)/chroots +WWTFTPDIR := $(TFTPDIR)/$(WAREWULF) +WWDOCDIR := $(DOCDIR)/$(WAREWULF) +WWDATADIR := $(DATADIR)/$(WAREWULF) +WWCLIENTDIR ?= /warewulf # auto installed tooling TOOLS_DIR := .tools @@ -51,7 +93,7 @@ GOPROXY ?= https://proxy.golang.org export GOPROXY # built tags needed for wwbuild binary -WW_BUILD_GO_BUILD_TAGS := containers_image_openpgp containers_image_ostree +WW_GO_BUILD_TAGS := containers_image_openpgp containers_image_ostree all: config vendor wwctl wwclient bash_completion.d man_pages @@ -68,10 +110,8 @@ $(GO_TOOLS_BIN): $(GOLANGCI_LINT): @curl -qq -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) $(GOLANGCI_LINT_VERSION) - setup: vendor $(TOOLS_DIR) setup_tools -# vendor vendor: go mod tidy -v go mod vendor @@ -81,27 +121,11 @@ $(TOOLS_DIR): # Pre-build steps for source, such as "go generate" config: - set -x ;\ - for i in `find . -type f -name "*.in" -not -path "./vendor/*"`; do \ - NAME=`echo $$i | sed -e 's,\.in,,'`; \ - sed -e 's,@BINDIR@,$(BINDIR),g' \ - -e 's,@SYSCONFDIR@,$(SYSCONFDIR),g' \ - -e 's,@LOCALSTATEDIR@,$(LOCALSTATEDIR),g' \ - -e 's,@PREFIX@,$(PREFIX),g' \ - -e 's,@DATADIR@,$(DATADIR),g' \ - -e 's,@MANDIR@,$(MANDIR),g' \ - -e 's,@SRVDIR@,$(SRVDIR),g' \ - -e 's,@TFTPDIR@,$(TFTPDIR),g' \ - -e 's,@FIREWALLDDIR@,$(FIREWALLDDIR),g' \ - -e 's,@SYSTEMDDIR@,$(SYSTEMDDIR),g' \ - -e 's,@BASH_COMPLETION@,$(BASH_COMPLETION),g' \ - -e 's,@WWOVERLAYDIR@,$(WWOVERLAYDIR),g' \ - -e 's,@WWCHROOTDIR@,$(WWCHROOTDIR),g' \ - -e 's,@WWPROVISIONDIR@,$(WWPROVISIONDIR),g' \ - -e 's,@VERSION@,$(VERSION),g' \ - -e 's,@WWCLIENTLOC@,$(WWCLIENTLOC),g' \ - -e 's,@RELEASE@,$(RELEASE),g' $$i > $$NAME; \ - done + # Store configuration for subsequent runs + printf " $(foreach V,$(VARLIST),$V := $(strip $($V))\n)" > Defaults.mk + # Global variable search and replace for all *.in files + find . -type f -name "*.in" -not -path "./vendor/*" \ + -exec sh -c 'sed -ne "$(foreach V,$(VARLIST),s,@$V@,$(strip $($V)),g;)p" $${0} > $${0%.in}' {} \; touch config rm_config: @@ -109,11 +133,10 @@ rm_config: genconfig: rm_config config - # Lint lint: setup_tools @echo Running golangci-lint... - @$(GOLANGCI_LINT) run --build-tags "$(WW_BUILD_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... + @$(GOLANGCI_LINT) run --build-tags "$(WW_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... vet: go vet ./... @@ -137,96 +160,82 @@ files: all install -d -m 0755 $(DESTDIR)$(BINDIR) install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR) install -d -m 0755 $(DESTDIR)$(WWPROVISIONDIR) - install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR) - install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/warewulf/bin/ - install -d -m 0755 $(DESTDIR)$(SYSCONFDIR)/warewulf/ - install -d -m 0755 $(DESTDIR)$(SYSCONFDIR)/warewulf/ipxe - install -d -m 0755 $(DESTDIR)$(TFTPDIR)/warewulf/ipxe/ - install -d -m 0755 $(DESTDIR)$(DATADIR)/warewulf/ipxe/ - install -d -m 0755 $(DESTDIR)$(BASH_COMPLETION) + install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR) + install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/ipxe + install -d -m 0755 $(DESTDIR)$(WWTFTPDIR)/ipxe/ + install -d -m 0755 $(DESTDIR)$(BASHCOMPDIR) install -d -m 0755 $(DESTDIR)$(MANDIR)/man1 + install -d -m 0755 $(DESTDIR)$(WWDOCDIR) install -d -m 0755 $(DESTDIR)$(FIREWALLDDIR) install -d -m 0755 $(DESTDIR)$(SYSTEMDDIR) - test -f $(DESTDIR)$(SYSCONFDIR)/warewulf/warewulf.conf || install -m 644 etc/warewulf.conf $(DESTDIR)$(SYSCONFDIR)/warewulf/ - test -f $(DESTDIR)$(SYSCONFDIR)/warewulf/nodes.conf || install -m 644 etc/nodes.conf $(DESTDIR)$(SYSCONFDIR)/warewulf/ - cp -r etc/examples $(DESTDIR)$(SYSCONFDIR)/warewulf/ - cp -r etc/ipxe $(DESTDIR)$(SYSCONFDIR)/warewulf/ + install -d -m 0755 $(DESTDIR)$(WWDATADIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf || install -m 644 etc/warewulf.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/nodes.conf || install -m 644 etc/nodes.conf $(DESTDIR)$(WWCONFIGDIR) + cp -r etc/examples $(DESTDIR)$(WWCONFIGDIR)/ + cp -r etc/ipxe $(DESTDIR)$(WWCONFIGDIR)/ cp -r overlays/* $(DESTDIR)$(WWOVERLAYDIR)/ chmod 755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/init - chmod 755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/warewulf/wwinit + find $(DESTDIR)$(WWOVERLAYDIR) -type f -name "*.in" -exec rm -f {} \; + chmod 755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwinit chmod 600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh* chmod 644 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh*.pub.ww install -m 0755 wwctl $(DESTDIR)$(BINDIR) - install -c -m 0644 include/firewalld/warewulf.xml $(DESTDIR)$(FIREWALLDDIR) - install -c -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR) - cp bash_completion.d/warewulf $(DESTDIR)$(BASH_COMPLETION) + install -m 0644 include/firewalld/warewulf.xml $(DESTDIR)$(FIREWALLDDIR) + install -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR) + install -m 0644 LICENSE.md $(DESTDIR)$(WWDOCDIR) + cp bash_completion.d/warewulf $(DESTDIR)$(BASHCOMPDIR) cp man_pages/* $(DESTDIR)$(MANDIR)/man1/ - install -c -m 0644 staticfiles/arm64.efi $(DESTDIR)$(DATADIR)/warewulf/ipxe - install -c -m 0644 staticfiles/i386.efi $(DESTDIR)$(DATADIR)/warewulf/ipxe - install -c -m 0644 staticfiles/i386.kpxe $(DESTDIR)$(DATADIR)/warewulf/ipxe - install -c -m 0644 staticfiles/x86.efi $(DESTDIR)$(DATADIR)/warewulf/ipxe + install -c -m 0644 staticfiles/arm64.efi $(DESTDIR)$(WWDATADIR)/ipxe + install -c -m 0644 staticfiles/i386.efi $(DESTDIR)$(WWDATADIR)/ipxe + install -c -m 0644 staticfiles/i386.kpxe $(DESTDIR)$(WWDATADIR)/ipxe + install -c -m 0644 staticfiles/x86.efi $(DESTDIR)$(WWDATADIR)/ipxe init: systemctl daemon-reload - cp -r tftpboot/* $(TFTPDIR)/warewulf/ipxe/ - restorecon -r $(TFTPDIR)/warewulf - -# Overlay file system has changed -#debfiles: debian -# chmod +x $(DESTDIR)$(WWROOT)/warewulf/overlays/system/debian/init -# chmod 600 $(DESTDIR)$(WWROOT)/warewulf/overlays/system/debian/etc/ssh/ssh* -# chmod 644 $(DESTDIR)$(WWROOT)/warewulf/overlays/system/debian/etc/ssh/ssh*.pub.ww -# mkdir -p $(DESTDIR)$(WWROOT)/warewulf/overlays/system/debian/warewulf/bin/ -# cp wwclient $(DESTDIR)$(WWROOT)/warewulf/overlays/system/debian/warewulf/bin/ + cp -r tftpboot/* $(WWTFTPDIR)/ipxe/ + restorecon -r $(WWTFTPDIR) wwctl: - cd cmd/wwctl; GOOS=linux go build -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../wwctl + cd cmd/wwctl; GOOS=linux go build -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o ../../wwctl wwclient: cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static \ -X 'github.com/hpcng/warewulf/internal/pkg/warewulfconf.ConfigFile=/etc/warewulf/warewulf.conf'" -o ../../wwclient install_wwclient: wwclient - install -m 0755 wwclient $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTLOC)/wwclient - + install -m 0755 wwclient $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwclient bash_completion: cd cmd/bash_completion && go build -ldflags="-X 'github.com/hpcng/warewulf/internal/pkg/warewulfconf.ConfigFile=./etc/warewulf.conf'\ -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf'"\ - -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../bash_completion + -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o ../../bash_completion bash_completion.d: bash_completion install -d -m 0755 bash_completion.d - ./bash_completion bash_completion.d/warewulf + ./bash_completion bash_completion.d/warewulf man_page: cd cmd/man_page && go build -ldflags="-X 'github.com/hpcng/warewulf/internal/pkg/warewulfconf.ConfigFile=./etc/warewulf.conf'\ -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf'"\ - -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../man_page + -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o ../../man_page man_pages: man_page install -d man_pages ./man_page ./man_pages cd man_pages; for i in wwctl*1; do echo "Compressing manpage: $$i"; gzip --force $$i; done -#config_defaults: -# cd cmd/config_defaults && go build -ldflags="-X 'github.com/hpcng/warewulf/internal/pkg/warewulfconf.ConfigFile=./etc/warewulf.conf' \ -# -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf' \ -# -X 'github.com/hpcng/warewulf/internal/pkg/warewulfconf.defaultDataStore=$(WWROOT)/warewulf'" \ -# -mod vendor -tags "$(WW_BUILD_GO_BUILD_TAGS)" -o ../../config_defaults - dist: vendor config - rm -rf .dist/warewulf-$(VERSION) - mkdir -p .dist/warewulf-$(VERSION) - cp -rap * .dist/warewulf-$(VERSION)/ - cd .dist; tar -czf ../warewulf-$(VERSION).tar.gz warewulf-$(VERSION) + rm -rf .dist/$(WAREWULF)-$(VERSION) + mkdir -p .dist/$(WAREWULF)-$(VERSION) + cp -rap * .dist/$(WAREWULF)-$(VERSION)/ + cd .dist; tar -czf ../$(WAREWULF)-$(VERSION).tar.gz $(WAREWULF)-$(VERSION) rm -rf .dist clean: rm -f wwclient rm -f wwctl rm -rf .dist - rm -f warewulf-$(VERSION).tar.gz + rm -f $(WAREWULF)-$(VERSION).tar.gz rm -f bash_completion rm -rf bash_completion.d rm -f man_page diff --git a/internal/app/wwclient/root.go b/internal/app/wwclient/root.go index e61534fd..8b6c590c 100644 --- a/internal/app/wwclient/root.go +++ b/internal/app/wwclient/root.go @@ -63,7 +63,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { return errors.New("found pidfile " + PIDFile + " not starting") } - if os.Args[0] == path.Join(buildconfig.WWCLIENTLOC(), "wwclient") { + if os.Args[0] == path.Join(buildconfig.WWCLIENTDIR(), "wwclient") { err := os.Chdir("/") if err != nil { wwlog.Printf(wwlog.ERROR, "failed to change dir: %s", err) @@ -75,7 +75,7 @@ func CobraRunE(cmd *cobra.Command, args []string) error { } else { fmt.Printf("Called via: %s\n", os.Args[0]) fmt.Printf("Runtime overlay is being put in '/warewulf/wwclient-test' rather than '/'\n") - fmt.Printf("For full functionality call with: %s\n", path.Join(buildconfig.WWCLIENTLOC(), "wwclient")) + fmt.Printf("For full functionality call with: %s\n", path.Join(buildconfig.WWCLIENTDIR(), "wwclient")) err := os.MkdirAll("/warewulf/wwclient-test", 0755) if err != nil { wwlog.Printf(wwlog.ERROR, "failed to create dir: %s", err) diff --git a/internal/app/wwctl/overlay/build/main.go b/internal/app/wwctl/overlay/build/main.go index 55515d35..916b4226 100644 --- a/internal/app/wwctl/overlay/build/main.go +++ b/internal/app/wwctl/overlay/build/main.go @@ -12,7 +12,7 @@ import ( func CobraRunE(cmd *cobra.Command, args []string) error { - if BuildHost || (!BuildHost && !BuildNodes) { + if BuildHost || (!BuildHost && !BuildNodes && len(args) == 0) { err := overlay.BuildHostOverlay() if err != nil { wwlog.Printf(wwlog.WARN, "host overlay could not be built: %s\n", err) diff --git a/internal/pkg/buildconfig/defaults.go b/internal/pkg/buildconfig/defaults.go index a648346b..fc714a3d 100644 --- a/internal/pkg/buildconfig/defaults.go +++ b/internal/pkg/buildconfig/defaults.go @@ -15,7 +15,7 @@ var ( wwprovisiondir string = "UNDEF" version string = "UNDEF" release string = "UNDEF" - wwclientloc string = "UNDEF" + wwclientdir string = "UNDEF" datadir string = "UNDEF" ) @@ -84,7 +84,7 @@ func RELEASE() string { return release } -func WWCLIENTLOC() string { - wwlog.Printf(wwlog.DEBUG, "WWCLIENTLOC = '%s'\n", wwclientloc) - return wwclientloc +func WWCLIENTDIR() string { + wwlog.Printf(wwlog.DEBUG, "WWCLIENTDIR = '%s'\n", wwclientdir) + return wwclientdir } diff --git a/internal/pkg/buildconfig/setconfigs.go.in b/internal/pkg/buildconfig/setconfigs.go.in index 43c9f87c..e10303e3 100644 --- a/internal/pkg/buildconfig/setconfigs.go.in +++ b/internal/pkg/buildconfig/setconfigs.go.in @@ -14,5 +14,5 @@ func init() { wwprovisiondir = "@WWPROVISIONDIR@" version = "@VERSION@" release = "@RELEASE@" - wwclientloc = "@WWCLIENTLOC@" + wwclientdir = "@WWCLIENTDIR@" } diff --git a/internal/pkg/node/methods.go b/internal/pkg/node/methods.go index a97fa895..58fb58e5 100644 --- a/internal/pkg/node/methods.go +++ b/internal/pkg/node/methods.go @@ -2,6 +2,9 @@ package node import ( "regexp" + "strings" + + "github.com/hpcng/warewulf/internal/pkg/util" ) /********** @@ -49,7 +52,6 @@ func (ent *Entry) Set(val string) { } else { ent.value = val } - } func (ent *Entry) SetB(val bool) { @@ -85,10 +87,10 @@ func (ent *Entry) SetDefault(val string) { } /********** - * - * Gets - * - *********/ +* +* Gets +* +*********/ func (ent *Entry) Get() string { if ent.value != "" { @@ -117,6 +119,25 @@ func (ent *Entry) GetReal() string { return ent.value } +/* +Returns a string slice, which is the combination of value, altvalue and def. +The elemtent in the slice are uniq. +*/ +func (ent *Entry) GetSlice() []string { + var retval []string + if ent.value != "" { + retval = util.SliceAppendUniq(retval, strings.Split(ent.value, ",")) + } + if ent.altvalue != "" { + retval = util.SliceAppendUniq(retval, strings.Split(ent.altvalue, ",")) + } + if ent.def != "" { + retval = util.SliceAppendUniq(retval, strings.Split(ent.def, ",")) + + } + return retval +} + /********** * * Misc diff --git a/internal/pkg/overlay/config.go b/internal/pkg/overlay/config.go index e033cb68..92879822 100644 --- a/internal/pkg/overlay/config.go +++ b/internal/pkg/overlay/config.go @@ -2,6 +2,7 @@ package overlay import ( "path" + "strings" "github.com/hpcng/warewulf/internal/pkg/buildconfig" ) @@ -14,6 +15,9 @@ func OverlaySourceDir(overlayName string) string { return path.Join(OverlaySourceTopDir(), overlayName) } -func OverlayImage(nodeName string, overlayName string) string { - return path.Join(buildconfig.WWPROVISIONDIR(), "overlays/", nodeName, overlayName+".img") +/* +Returns the overlay name of the image for a given node +*/ +func OverlayImage(nodeName string, overlayName []string) string { + return path.Join(buildconfig.WWPROVISIONDIR(), "overlays/", nodeName, strings.Join(overlayName, "-")+".img") } diff --git a/internal/pkg/overlay/overlay.go b/internal/pkg/overlay/overlay.go index e91169b8..e9a5737c 100644 --- a/internal/pkg/overlay/overlay.go +++ b/internal/pkg/overlay/overlay.go @@ -41,20 +41,21 @@ func FindRuntimeOverlays() ([]string, error) { } */ +/* +Build all overlays (runtime and generic) for a node +*/ func BuildAllOverlays(nodes []node.NodeInfo) error { for _, n := range nodes { var overlays []string - overlays = append(overlays, n.SystemOverlay.Get()) - overlays = append(overlays, n.RuntimeOverlay.Get()) + overlays = append(overlays, n.SystemOverlay.GetSlice()...) + overlays = append(overlays, n.RuntimeOverlay.GetSlice()...) wwlog.Printf(wwlog.INFO, "Building overlays for %s: [%s]\n", n.Id.Get(), strings.Join(overlays, ", ")) - for _, overlay := range overlays { - err := BuildOverlay(n, overlay) - if err != nil { - return errors.Wrap(err, "could not build overlay "+n.Id.Get()+"/"+overlay+".img") - } + err := BuildOverlay(n, overlays) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("could not build overlays %v for nide %s\n", overlays, n.Id.Get())) } } return nil @@ -66,7 +67,7 @@ func BuildSpecificOverlays(nodes []node.NodeInfo, overlayName string) error { for _, n := range nodes { wwlog.Printf(wwlog.INFO, "Building overlay for %s: %s\n", n.Id.Get(), overlayName) - err := BuildOverlay(n, overlayName) + err := BuildOverlay(n, []string{overlayName}) if err != nil { return errors.Wrap(err, "could not build overlay "+n.Id.Get()+"/"+overlayName+".img") } @@ -85,9 +86,12 @@ func BuildHostOverlay() error { wwlog.Printf(wwlog.INFO, "Building overlay for %s: host\n", hostname) idEntry.Set(hostname) host.Id = idEntry - return BuildOverlay(host, "host") + return BuildOverlay(host, []string{"host"}) } +/* +Get all overlays present in warewulf +*/ func FindOverlays() ([]string, error) { var ret []string var files []os.FileInfo @@ -119,33 +123,32 @@ func OverlayInit(overlayName string) error { return err } -func BuildOverlay(nodeInfo node.NodeInfo, overlayName string) error { +/* +Build the given overlays for a node. If the overlayName is = "host", the host overlay is build +*/ +func BuildOverlay(nodeInfo node.NodeInfo, overlayNames []string) error { + if len(overlayNames) == 0 { + return errors.New("At least one valid overlay is needed to build for a node\n") + } controller, err := warewulfconf.New() if err != nil { wwlog.Printf(wwlog.ERROR, "%s\n", err) os.Exit(1) } - nodeDB, _ := node.New() allNodes, _ := nodeDB.FindAllNodes() - var tstruct TemplateStruct - OverlaySourceDir := OverlaySourceDir(overlayName) var overlayImage string = "/" - if overlayName != "host" { - overlayImage = OverlayImage(nodeInfo.Id.Get(), overlayName) + if overlayNames[0] != "host" { + overlayImage = OverlayImage(nodeInfo.Id.Get(), overlayNames) } + var tstruct TemplateStruct OverlayImageDir := path.Dir(overlayImage) - if !util.ValidString(overlayName, "^[a-zA-Z0-9-._:]+$") { - return errors.New("overlay name contains illegal characters: " + overlayName) - } - - wwlog.Printf(wwlog.DEBUG, "Checking to see if overlay directory exists: %s\n", OverlaySourceDir) - if !util.IsDir(OverlaySourceDir) { - return errors.New("overlay does not exist: " + overlayName) + if !util.ValidString(strings.Join(overlayNames, ""), "^[a-zA-Z0-9-._:]+$") { + return errors.New(fmt.Sprintf("overlay names contains illegal characters: %v", overlayNames)) } var destDir = "/" - if overlayName != "host" { + if overlayNames[0] != "host" { err = os.MkdirAll(OverlayImageDir, 0755) if err == nil { wwlog.Printf(wwlog.DEBUG, "Created parent directory for Overlay Images: %s\n", OverlayImageDir) @@ -161,7 +164,7 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayName string) error { } } - wwlog.Printf(wwlog.VERBOSE, "Processing node/overlay: %s/%s\n", nodeInfo.Id.Get(), overlayName) + wwlog.Printf(wwlog.VERBOSE, "Processing node/overlay: %s/%s\n", nodeInfo.Id.Get(), strings.Join(overlayNames, "-")) tstruct.Id = nodeInfo.Id.Get() tstruct.Hostname = nodeInfo.Id.Get() @@ -225,130 +228,138 @@ func BuildOverlay(nodeInfo node.NodeInfo, overlayName string) error { tstruct.BuildHost = hostname dt := time.Now() tstruct.BuildTime = dt.Format("01-02-2006 15:04:05 MST") - wwlog.Printf(wwlog.DEBUG, "Changing directory to OverlayDir: %s\n", OverlaySourceDir) - err = os.Chdir(OverlaySourceDir) - if err != nil { - return errors.Wrap(err, "could not change directory to overlay dir") - } - - wwlog.Printf(wwlog.VERBOSE, "Walking the overlay structure: %s\n", OverlaySourceDir) - err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { + for _, overlayName := range overlayNames { + wwlog.Printf(wwlog.DEBUG, "Staring to build overlay %s\nChanging directory to OverlayDir: %s\n", + overlayName, OverlaySourceDir) + OverlaySourceDir := OverlaySourceDir(overlayName) + err = os.Chdir(OverlaySourceDir) if err != nil { - return err + return errors.Wrap(err, "could not change directory to overlay dir") + } + wwlog.Printf(wwlog.DEBUG, "Checking to see if overlay directory exists: %s\n", OverlaySourceDir) + if !util.IsDir(OverlaySourceDir) { + return errors.New("overlay does not exist: " + overlayName) } - wwlog.Printf(wwlog.DEBUG, "Found overlay file: %s\n", location) - - if info.IsDir() { - wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) - - err = os.MkdirAll(path.Join(destDir, location), info.Mode()) + wwlog.Printf(wwlog.VERBOSE, "Walking the overlay structure: %s\n", OverlaySourceDir) + err = filepath.Walk(".", func(location string, info os.FileInfo, err error) error { if err != nil { - return errors.Wrap(err, "could not create directory within overlay") - } - err = util.CopyUIDGID(location, path.Join(destDir, location)) - if err != nil { - return errors.Wrap(err, "failed setting permissions on overlay directory") + return err } - wwlog.Printf(wwlog.DEBUG, "Created directory in overlay: %s\n", location) + wwlog.Printf(wwlog.DEBUG, "Found overlay file: %s\n", location) - } else if filepath.Ext(location) == ".ww" { - tstruct.BuildSource = path.Join(OverlaySourceDir, location) - wwlog.Printf(wwlog.VERBOSE, "Evaluating overlay template file: %s\n", location) + if info.IsDir() { + wwlog.Printf(wwlog.DEBUG, "Found directory: %s\n", location) - destFile := strings.TrimSuffix(location, ".ww") - ErrorAbort := errors.New("abort_template") - ErrorNoBackup := errors.New("nobackup_template") - tmpl, err := template.New(path.Base(location)).Option("missingkey=default").Funcs(template.FuncMap{ - // TODO: Fix for missingkey=zero - "Include": templateFileInclude, - "IncludeFrom": templateContainerFileInclude, - "IncludeBlock": templateFileBlock, - "inc": func(i int) int { return i + 1 }, - "dec": func(i int) int { return i - 1 }, - "abort": func() (string, error) { return "", ErrorAbort }, - "nobackup": func() (string, error) { return "", ErrorNoBackup }, - // }).ParseGlob(path.Join(OverlayDir, destFile+".ww*")) - }).ParseGlob(location) - if err != nil { - return errors.Wrap(err, "could not parse template "+location) - } - var buffer bytes.Buffer - backupFile := true - writeFile := true - err = tmpl.Execute(&buffer, tstruct) - if err != nil { - // complicated workarround as error is not exported correctly: https://github.com/golang/go/issues/34201 - if strings.Contains(fmt.Sprint(err), "abort_template") { - wwlog.Printf(wwlog.VERBOSE, "Aborting template file due to abort call in template: %s\n", location) - writeFile = false - } else if strings.Contains(fmt.Sprint(err), "nobackup_template") { - backupFile = false - } else { - return errors.Wrap(err, "could not execute template") + err = os.MkdirAll(path.Join(destDir, location), info.Mode()) + if err != nil { + return errors.Wrap(err, "could not create directory within overlay") } - } - if writeFile { - if backupFile { - if !util.IsFile(path.Join(destDir, destFile+".wwbackup")) && util.IsFile(path.Join(destDir, destFile)) { - err = util.CopyFile(path.Join(destDir, destFile), path.Join(destDir, destFile+".wwbackup")) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) + err = util.CopyUIDGID(location, path.Join(destDir, location)) + if err != nil { + return errors.Wrap(err, "failed setting permissions on overlay directory") + } + + wwlog.Printf(wwlog.DEBUG, "Created directory in overlay: %s\n", location) + + } else if filepath.Ext(location) == ".ww" { + tstruct.BuildSource = path.Join(OverlaySourceDir, location) + wwlog.Printf(wwlog.VERBOSE, "Evaluating overlay template file: %s\n", location) + + destFile := strings.TrimSuffix(location, ".ww") + ErrorAbort := errors.New("abort_template") + ErrorNoBackup := errors.New("nobackup_template") + tmpl, err := template.New(path.Base(location)).Option("missingkey=default").Funcs(template.FuncMap{ + // TODO: Fix for missingkey=zero + "Include": templateFileInclude, + "IncludeFrom": templateContainerFileInclude, + "IncludeBlock": templateFileBlock, + "inc": func(i int) int { return i + 1 }, + "dec": func(i int) int { return i - 1 }, + "abort": func() (string, error) { return "", ErrorAbort }, + "nobackup": func() (string, error) { return "", ErrorNoBackup }, + // }).ParseGlob(path.Join(OverlayDir, destFile+".ww*")) + }).ParseGlob(location) + if err != nil { + return errors.Wrap(err, "could not parse template "+location) + } + var buffer bytes.Buffer + backupFile := true + writeFile := true + err = tmpl.Execute(&buffer, tstruct) + if err != nil { + // complicated workarround as error is not exported correctly: https://github.com/golang/go/issues/34201 + if strings.Contains(fmt.Sprint(err), "abort_template") { + wwlog.Printf(wwlog.VERBOSE, "Aborting template file due to abort call in template: %s\n", location) + writeFile = false + } else if strings.Contains(fmt.Sprint(err), "nobackup_template") { + backupFile = false + } else { + return errors.Wrap(err, "could not execute template") + } + } + if writeFile { + if backupFile { + if !util.IsFile(path.Join(destDir, destFile+".wwbackup")) && util.IsFile(path.Join(destDir, destFile)) { + err = util.CopyFile(path.Join(destDir, destFile), path.Join(destDir, destFile+".wwbackup")) + if err != nil { + wwlog.Printf(wwlog.ERROR, "%s\n", err) + } } + + } + w, err := os.OpenFile(path.Join(destDir, destFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) + if err != nil { + return errors.Wrap(err, "could not open new file for template") + } + defer w.Close() + + _, err = buffer.WriteTo(w) + + if err != nil { + return errors.Wrap(err, "could not write file from template") } + err = util.CopyUIDGID(location, path.Join(destDir, destFile)) + if err != nil { + return errors.Wrap(err, "failed setting permissions on template output file") + } + + wwlog.Printf(wwlog.DEBUG, "Wrote template file into overlay: %s\n", destFile) + + // } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b { + // wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location) } - w, err := os.OpenFile(path.Join(destDir, destFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) + } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { + wwlog.Printf(wwlog.DEBUG, "Found symlink %s\n", location) + destination, err := os.Readlink(location) if err != nil { - return errors.Wrap(err, "could not open new file for template") + wwlog.Printf(wwlog.ERROR, "%s\n", err) } - defer w.Close() - - _, err = buffer.WriteTo(w) - + err = os.Symlink(destination, path.Join(destDir, location)) if err != nil { - return errors.Wrap(err, "could not write file from template") + wwlog.Printf(wwlog.ERROR, "%s\n", err) } - - err = util.CopyUIDGID(location, path.Join(destDir, destFile)) - if err != nil { - return errors.Wrap(err, "failed setting permissions on template output file") - } - - wwlog.Printf(wwlog.DEBUG, "Wrote template file into overlay: %s\n", destFile) - - // } else if b, _ := regexp.MatchString(`\.ww[a-zA-Z0-9\-\._]*$`, location); b { - // wwlog.Printf(wwlog.DEBUG, "Ignoring WW template file: %s\n", location) - } - } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { - wwlog.Printf(wwlog.DEBUG, "Found symlink %s\n", location) - destination, err := os.Readlink(location) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - } - err = os.Symlink(destination, path.Join(destDir, location)) - if err != nil { - wwlog.Printf(wwlog.ERROR, "%s\n", err) - } - } else { - - err := util.CopyFile(location, path.Join(destDir, location)) - if err == nil { - wwlog.Printf(wwlog.DEBUG, "Copied file into overlay: %s\n", location) } else { - return errors.Wrap(err, "could not copy file into overlay") - } - } - return nil - }) - if err != nil { - return errors.Wrap(err, "failed to build overlay working directory") + err := util.CopyFile(location, path.Join(destDir, location)) + if err == nil { + wwlog.Printf(wwlog.DEBUG, "Copied file into overlay: %s\n", location) + } else { + return errors.Wrap(err, "could not copy file into overlay") + } + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "failed to build overlay working directory") + } } - wwlog.Printf(wwlog.DEBUG, "Finished generating overlay working directory for: %s/%s\n", nodeInfo.Id.Get(), overlayName) - if overlayName != "host" { + wwlog.Printf(wwlog.DEBUG, "Finished generating overlay working directory for: %s/%v\n", nodeInfo.Id.Get(), overlayNames) + if overlayNames[0] != "host" { compressor, err := exec.LookPath("pigz") if err != nil { wwlog.Printf(wwlog.DEBUG, "Could not locate PIGZ, using GZIP\n") diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index c4f121fa..8ac47207 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -203,6 +203,9 @@ func SliceRemoveElement(array []string, remove string) []string { return ret } +/* +Adds a string, to string slice if the given string is not present in the slice. +*/ func SliceAddUniqueElement(array []string, add string) []string { var ret []string var found bool @@ -222,6 +225,17 @@ func SliceAddUniqueElement(array []string, add string) []string { return ret } +/* +Appends a string slice to another slice. Guarantess that the elements are uniq. +*/ +func SliceAppendUniq(array []string, add []string) []string { + var ret []string + for _, r := range add { + ret = SliceAddUniqueElement(array, r) + } + return ret +} + func SystemdStart(systemdName string) error { startCmd := fmt.Sprintf("systemctl restart %s", systemdName) enableCmd := fmt.Sprintf("systemctl enable %s", systemdName) diff --git a/internal/pkg/warewulfd/runtime.go b/internal/pkg/warewulfd/runtime.go index 69d5e4c0..689c4107 100644 --- a/internal/pkg/warewulfd/runtime.go +++ b/internal/pkg/warewulfd/runtime.go @@ -46,15 +46,19 @@ func RuntimeOverlaySend(w http.ResponseWriter, req *http.Request) { } } - if node.RuntimeOverlay.Defined() { - fileName := overlay.OverlayImage(node.Id.Get(), node.RuntimeOverlay.Get()) + if len(node.RuntimeOverlay.GetSlice()) != 0 { + fileName := overlay.OverlayImage(node.Id.Get(), node.RuntimeOverlay.GetSlice()) - updateStatus(node.Id.Get(), "RUNTIME_OVERLAY", node.RuntimeOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + updateStatus(node.Id.Get(), "RUNTIME_OVERLAY", node.RuntimeOverlay.Get(), strings.Split(req.RemoteAddr, ":")[0]) if conf.Warewulf.AutobuildOverlays { - if !util.IsFile(fileName) || util.PathIsNewer(fileName, nodepkg.ConfigFile) || util.PathIsNewer(fileName, overlay.OverlaySourceDir(node.RuntimeOverlay.Get())) { + oneoverlaynewer := false + for _, overlayname := range node.RuntimeOverlay.GetSlice() { + oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(fileName, overlay.OverlaySourceDir(overlayname)) + } + if !util.IsFile(fileName) || util.PathIsNewer(fileName, nodepkg.ConfigFile) || oneoverlaynewer { daemonLogf("BUILD: %15s: Runtime Overlay\n", node.Id.Get()) - _ = overlay.BuildOverlay(node, node.RuntimeOverlay.Get()) + _ = overlay.BuildOverlay(node, node.RuntimeOverlay.GetSlice()) } } diff --git a/internal/pkg/warewulfd/system.go b/internal/pkg/warewulfd/system.go index 68078c75..7574f00b 100644 --- a/internal/pkg/warewulfd/system.go +++ b/internal/pkg/warewulfd/system.go @@ -38,15 +38,19 @@ func SystemOverlaySend(w http.ResponseWriter, req *http.Request) { return } - if node.SystemOverlay.Defined() { - fileName := overlay.OverlayImage(node.Id.Get(), node.SystemOverlay.Get()) + if len(node.SystemOverlay.GetSlice()) != 0 { + fileName := overlay.OverlayImage(node.Id.Get(), node.SystemOverlay.GetSlice()) - updateStatus(node.Id.Get(), "SYSTEM_OVERLAY", node.SystemOverlay.Get()+".img", strings.Split(req.RemoteAddr, ":")[0]) + updateStatus(node.Id.Get(), "SYSTEM_OVERLAY", node.SystemOverlay.Get(), strings.Split(req.RemoteAddr, ":")[0]) if conf.Warewulf.AutobuildOverlays { - if !util.IsFile(fileName) || util.PathIsNewer(fileName, nodepkg.ConfigFile) || util.PathIsNewer(fileName, overlay.OverlaySourceDir(node.SystemOverlay.Get())) { + oneoverlaynewer := false + for _, overlayname := range node.RuntimeOverlay.GetSlice() { + oneoverlaynewer = oneoverlaynewer || util.PathIsNewer(fileName, overlay.OverlaySourceDir(overlayname)) + } + if !util.IsFile(fileName) || util.PathIsNewer(fileName, nodepkg.ConfigFile) || oneoverlaynewer { daemonLogf("BUILD: %15s: System Overlay\n", node.Id.Get()) - _ = overlay.BuildOverlay(node, node.SystemOverlay.Get()) + _ = overlay.BuildOverlay(node, node.SystemOverlay.GetSlice()) } } diff --git a/warewulf.spec.in b/warewulf.spec.in index e80b03c5..a4b2dbac 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -1,13 +1,16 @@ -%global wwgroup warewulf -%if 0%{?sle_version} -%global tftpdir "/srv/tftpboot" -%global srvdir "/srv" -%else -%global tftpdir "/var/lib/tftpboot" -%global srvdir "%{_localstatedir}/warewulf/srv" -%endif %define debug_package %{nil} +%if 0%{?sle_version} +%global tftpdir /srv/tftpboot +%global srvdir /srv +%else +# Assume Fedora-based OS if not SUSE-based +%global tftpdir /var/lib/tftpboot +%global srvdir %{_sharedstatedir} +%endif + +%global wwgroup warewulf + Name: warewulf Summary: A provisioning system for large clusters of bare metal and/or virtual systems Version: @VERSION@ @@ -27,17 +30,18 @@ Conflicts: warewulf-ipmi BuildRequires: make -%if 0%{?rhel} -BuildRequires: systemd -BuildRequires: golang -Requires: tftp-server -Requires: nfs-utils -%else -# sle_version +%if 0%{?sle_version} BuildRequires: systemd-rpm-macros BuildRequires: go +BuildRequires: firewall-macros Requires: tftp Requires: nfs-kernel-server +%else +BuildRequires: systemd +BuildRequires: golang +BuildRequires: firewalld-filesystem +Requires: tftp-server +Requires: nfs-utils %endif %if 0%{?rhel} >= 8 || 0%{?sle_version} @@ -57,30 +61,37 @@ system for large clusters of bare metal and/or virtual systems. %build -make genconfig PREFIX=/usr \ - SRVDIR=%{srvdir} \ - LOCALSTATEDIR=%{_localstatedir} \ +# Install to sharedstatedir by redirecting LOCALSTATEDIR +make genconfig \ + PREFIX=%{_prefix} \ + BINDIR=%{_bindir} \ SYSCONFDIR=%{_sysconfdir} \ DATADIR=%{_datadir} \ + LOCALSTATEDIR=%{_sharedstatedir} \ + SHAREDSTATEDIR=%{_sharedstatedir} \ MANDIR=%{_mandir} \ - BINDIR=%{_bindir} \ + INFODIR=%{_infodir} \ + DOCDIR=%{_docdir} \ + SRVDIR=%{srvdir} \ TFTPDIR=%{tftpdir} \ SYSTEMDDIR=%{_unitdir} \ - MANDIR=%{_mandir} \ - BASH_COMPLETION=/etc/bash_completion.d/ \ - FIREWALLDDIR=/usr/lib/firewalld/services - + BASHCOMPDIR=/etc/bash_completion.d/ \ + FIREWALLDDIR=/usr/lib/firewalld/services \ + WWCLIENTDIR=/warewulf make + %install make install DESTDIR=%{buildroot} + %pre getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %post %systemd_post warewulfd.service +%firewalld_reload %preun @@ -89,19 +100,20 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %postun %systemd_postun_with_restart warewulfd.service +%firewalld_reload %files %defattr(-, root, %{wwgroup}) -%dir %{_sysconfdir}/%{name} -%config(noreplace) %{_sysconfdir}/%{name}/* -%config(noreplace) %attr(0640,-,-) %{_sysconfdir}/%{name}/nodes.conf +%dir %{_sysconfdir}/warewulf +%config(noreplace) %{_sysconfdir}/warewulf/* +%config(noreplace) %attr(0640,-,-) %{_sysconfdir}/warewulf/nodes.conf %{_sysconfdir}/bash_completion.d/warewulf -%dir %{_localstatedir}/warewulf -%{_localstatedir}/warewulf/chroots -%{_localstatedir}/warewulf/overlays -%{srvdir} +%dir %{_sharedstatedir}/warewulf +%{_sharedstatedir}/warewulf/chroots +%{_sharedstatedir}/warewulf/overlays +%{srvdir}/warewulf %attr(-, root, root) %{_bindir}/wwctl %attr(-, root, root) %{_prefix}/lib/firewalld/services/warewulf.xml @@ -109,7 +121,14 @@ getent group %{wwgroup} >/dev/null || groupadd -r %{wwgroup} %attr(-, root, root) %{_mandir}/man1/wwctl* %attr(-, root, root) %{_datadir}/warewulf +%license %{_docdir}/warewulf/LICENSE.md + + %changelog +* Wed Jan 26 2022 Jeremy Siadal - 4.2.0-1 +- Add license install +- Updates for RH and SUSE RPM guidelines + * Sat Jan 15 2022 Gregory Kurtzer - 4.2.0-1 - Integrated genconfig Make options - Cleaned up SPEC to use default RPM macros