From 94e80b17d7514c631e289a2323c11627c63a4511 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 01:21:33 -0600 Subject: [PATCH 01/24] Split Makefile variables into a separate file Signed-off-by: Jonathon Anderson --- Makefile | 100 +-------------------------------------------------- Variables.mk | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 99 deletions(-) create mode 100644 Variables.mk diff --git a/Makefile b/Makefile index 2cb29c9c..ea64242a 100644 --- a/Makefile +++ b/Makefile @@ -1,104 +1,6 @@ .PHONY: all clean contclean --include Defaults.mk - -# Linux distro (try and set to /etc/os-release ID) -OS_REL := $(shell sed -n "s/^ID\s*=\s*['"\""]\(.*\)['"\""]/\1/p" /etc/os-release) -OS ?= $(OS_REL) - -# List of variables to save and replace in files -VARLIST := OS - -# Project Information -VARLIST += WAREWULF VERSION RELEASE -WAREWULF ?= warewulf -VERSION ?= 4.4.0 -GIT_TAG := $(shell test -e .git && git log -1 --format="%h") - -ifdef GIT_TAG - ifdef $(filter $(OS),ubuntu debian) - RELEASE ?= 1.git_$(subst -,+,$(GIT_TAG)) - else - RELEASE ?= 1.git_$(subst -,_,$(GIT_TAG)) - endif -else - RELEASE ?= 1 -endif - -# Use LSB-compliant paths if OS is known -ifneq ($(OS),) - USE_LSB_PATHS := true -endif - -# Always default to GNU autotools default paths if PREFIX has been redefined -ifdef PREFIX - USE_LSB_PATHS := false -endif - -# System directory paths -VARLIST += PREFIX BINDIR SYSCONFDIR SRVDIR DATADIR MANDIR DOCDIR LOCALSTATEDIR -PREFIX ?= /usr/local -BINDIR ?= $(PREFIX)/bin -SYSCONFDIR ?= $(PREFIX)/etc -DATADIR ?= $(PREFIX)/share -MANDIR ?= $(DATADIR)/man -DOCDIR ?= $(DATADIR)/doc - -ifeq ($(USE_LSB_PATHS),true) - SRVDIR ?= /srv - LOCALSTATEDIR ?= /var/local -else - SRVDIR ?= $(PREFIX)/srv - LOCALSTATEDIR ?= $(PREFIX)/var -endif - -# OS-Specific Service Locations -VARLIST += TFTPDIR FIREWALLDDIR SYSTEMDDIR BASHCOMPDIR -SYSTEMDDIR ?= /usr/lib/systemd/system -BASHCOMPDIR ?= /etc/bash_completion.d -FIREWALLDDIR ?= /usr/lib/firewalld/services -ifeq ($(OS),suse) - TFTPDIR ?= /srv/tftpboot -endif -ifeq ($(OS),ubuntu) - TFTPDIR ?= /srv/tftp -endif -# Default to Red Hat / Rocky Linux -TFTPDIR ?= /var/lib/tftpboot - -# Warewulf directory paths -VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR WWDATADIR -WWCONFIGDIR := $(SYSCONFDIR)/$(WAREWULF) -WWPROVISIONDIR := $(SRVDIR)/$(WAREWULF) -WWOVERLAYDIR := $(LOCALSTATEDIR)/$(WAREWULF)/overlays -WWCHROOTDIR := $(LOCALSTATEDIR)/$(WAREWULF)/chroots -WWTFTPDIR := $(TFTPDIR)/$(WAREWULF) -WWDOCDIR := $(DOCDIR)/$(WAREWULF) -WWDATADIR := $(DATADIR)/$(WAREWULF) -WWCLIENTDIR ?= /warewulf - -# auto installed tooling -TOOLS_DIR := .tools -TOOLS_BIN := $(TOOLS_DIR)/bin -CONFIG := $(shell pwd) - -# tools -GO_TOOLS_BIN := $(addprefix $(TOOLS_BIN)/, $(notdir $(GO_TOOLS))) -GO_TOOLS_VENDOR := $(addprefix vendor/, $(GO_TOOLS)) -GOLANGCI_LINT := $(TOOLS_BIN)/golangci-lint -GOLANGCI_LINT_VERSION := v1.53.2 - -# helper functions -godeps=$(shell go list -deps -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}' $(1) | sed "s%${PWD}/%%g") -WWCTL_DEPS:=$(call godeps,cmd/wwctl/main.go) -WWCLIENT_DEPS:=$(call godeps,cmd/wwclient/main.go) - -# use GOPROXY for older git clients and speed up downloads -GOPROXY ?= https://proxy.golang.org -export GOPROXY - -# built tags needed for wwbuild binary -WW_GO_BUILD_TAGS := containers_image_openpgp containers_image_ostree +include Variables.mk # Default target all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird diff --git a/Variables.mk b/Variables.mk new file mode 100644 index 00000000..29ef1924 --- /dev/null +++ b/Variables.mk @@ -0,0 +1,99 @@ +-include Defaults.mk + +# Linux distro (try and set to /etc/os-release ID) +OS_REL := $(shell sed -n "s/^ID\s*=\s*['"\""]\(.*\)['"\""]/\1/p" /etc/os-release) +OS ?= $(OS_REL) + +# List of variables to save and replace in files +VARLIST := OS + +# Project Information +VARLIST += WAREWULF VERSION RELEASE +WAREWULF ?= warewulf +VERSION ?= 4.4.0 +GIT_TAG := $(shell test -e .git && git log -1 --format="%h") + +ifdef GIT_TAG + ifdef $(filter $(OS),ubuntu debian) + RELEASE ?= 1.git_$(subst -,+,$(GIT_TAG)) + else + RELEASE ?= 1.git_$(subst -,_,$(GIT_TAG)) + endif +else + RELEASE ?= 1 +endif + +# Use LSB-compliant paths if OS is known +ifneq ($(OS),) + USE_LSB_PATHS := true +endif + +# Always default to GNU autotools default paths if PREFIX has been redefined +ifdef PREFIX + USE_LSB_PATHS := false +endif + +# System directory paths +VARLIST += PREFIX BINDIR SYSCONFDIR SRVDIR DATADIR MANDIR DOCDIR LOCALSTATEDIR +PREFIX ?= /usr/local +BINDIR ?= $(PREFIX)/bin +SYSCONFDIR ?= $(PREFIX)/etc +DATADIR ?= $(PREFIX)/share +MANDIR ?= $(DATADIR)/man +DOCDIR ?= $(DATADIR)/doc + +ifeq ($(USE_LSB_PATHS),true) + SRVDIR ?= /srv + LOCALSTATEDIR ?= /var/local +else + SRVDIR ?= $(PREFIX)/srv + LOCALSTATEDIR ?= $(PREFIX)/var +endif + +# OS-Specific Service Locations +VARLIST += TFTPDIR FIREWALLDDIR SYSTEMDDIR BASHCOMPDIR +SYSTEMDDIR ?= /usr/lib/systemd/system +BASHCOMPDIR ?= /etc/bash_completion.d +FIREWALLDDIR ?= /usr/lib/firewalld/services +ifeq ($(OS),suse) + TFTPDIR ?= /srv/tftpboot +endif +ifeq ($(OS),ubuntu) + TFTPDIR ?= /srv/tftp +endif +# Default to Red Hat / Rocky Linux +TFTPDIR ?= /var/lib/tftpboot + +# Warewulf directory paths +VARLIST += WWCLIENTDIR WWCONFIGDIR WWPROVISIONDIR WWOVERLAYDIR WWCHROOTDIR WWTFTPDIR WWDOCDIR WWDATADIR +WWCONFIGDIR := $(SYSCONFDIR)/$(WAREWULF) +WWPROVISIONDIR := $(SRVDIR)/$(WAREWULF) +WWOVERLAYDIR := $(LOCALSTATEDIR)/$(WAREWULF)/overlays +WWCHROOTDIR := $(LOCALSTATEDIR)/$(WAREWULF)/chroots +WWTFTPDIR := $(TFTPDIR)/$(WAREWULF) +WWDOCDIR := $(DOCDIR)/$(WAREWULF) +WWDATADIR := $(DATADIR)/$(WAREWULF) +WWCLIENTDIR ?= /warewulf + +# auto installed tooling +TOOLS_DIR := .tools +TOOLS_BIN := $(TOOLS_DIR)/bin +CONFIG := $(shell pwd) + +# tools +GO_TOOLS_BIN := $(addprefix $(TOOLS_BIN)/, $(notdir $(GO_TOOLS))) +GO_TOOLS_VENDOR := $(addprefix vendor/, $(GO_TOOLS)) +GOLANGCI_LINT := $(TOOLS_BIN)/golangci-lint +GOLANGCI_LINT_VERSION := v1.53.2 + +# helper functions +godeps=$(shell go list -deps -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}' $(1) | sed "s%${PWD}/%%g") +WWCTL_DEPS:=$(call godeps,cmd/wwctl/main.go) +WWCLIENT_DEPS:=$(call godeps,cmd/wwclient/main.go) + +# use GOPROXY for older git clients and speed up downloads +GOPROXY ?= https://proxy.golang.org +export GOPROXY + +# built tags needed for wwbuild binary +WW_GO_BUILD_TAGS := containers_image_openpgp containers_image_ostree From a8a68e47adce41068ca34a50c1146a29d72700a1 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 01:33:02 -0600 Subject: [PATCH 02/24] Change Warewulf version to 4.5.x Future releases from the development branch will be 4.5.x. Additional 4.4.x versions would be tagged from the 4.4.x branch. Signed-off-by: Jonathon Anderson --- Variables.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Variables.mk b/Variables.mk index 29ef1924..43e28ff6 100644 --- a/Variables.mk +++ b/Variables.mk @@ -10,7 +10,7 @@ VARLIST := OS # Project Information VARLIST += WAREWULF VERSION RELEASE WAREWULF ?= warewulf -VERSION ?= 4.4.0 +VERSION ?= 4.5.x GIT_TAG := $(shell test -e .git && git log -1 --format="%h") ifdef GIT_TAG From 315b78bcce77def5b582bf99ee86e031e8307141 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 01:37:19 -0600 Subject: [PATCH 03/24] Remove echo and hidden commands from Makefile Signed-off-by: Jonathon Anderson --- Makefile | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index ea64242a..958e700a 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,11 @@ setup_tools: $(GO_TOOLS_BIN) $(GOLANGCI_LINT) # install go tools into TOOLS_BIN $(GO_TOOLS_BIN): - @GOBIN="$(PWD)/$(TOOLS_BIN)" go install -mod=vendor $(GO_TOOLS) + GOBIN="$(PWD)/$(TOOLS_BIN)" go install -mod=vendor $(GO_TOOLS) # install golangci-lint into TOOLS_BIN $(GOLANGCI_LINT): - @curl -qq -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) $(GOLANGCI_LINT_VERSION) + curl -qq -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) $(GOLANGCI_LINT_VERSION) setup: vendor $(TOOLS_DIR) setup_tools @@ -28,7 +28,7 @@ ifndef OFFLINE_BUILD endif $(TOOLS_DIR): - @mkdir -p $@ + mkdir -p $@ # Pre-build steps for source, such as "go generate" config: @@ -46,8 +46,7 @@ genconfig: rm_config config # Lint lint: setup_tools - @echo Running golangci-lint... - @$(GOLANGCI_LINT) run --build-tags "$(WW_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... + $(GOLANGCI_LINT) run --build-tags "$(WW_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... vet: go vet ./... @@ -119,21 +118,18 @@ init: restorecon -r $(WWTFTPDIR) wwctl: config vendor $(WWCTL_DEPS) - @echo Building "$@" - @cd cmd/wwctl; GOOS=linux go build -mod vendor -tags "$(WW_GO_BUILD_TAGS)" \ + cd cmd/wwctl; GOOS=linux go build -mod vendor -tags "$(WW_GO_BUILD_TAGS)" \ -o ../../wwctl wwclient: config vendor $(WWCLIENT_DEPS) - @echo Building "$@" - @cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static" \ + cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static" \ -o ../../wwclient man_pages: wwctl - @install -d man_pages - @./wwctl --emptyconf genconfig man man_pages - @cp docs/man/man5/*.5 ./man_pages/ - @echo -n "Compressing manpage: " - @cd man_pages; for i in wwctl*1 *.5; do gzip --force $$i; echo -n "$$i "; done; echo + install -d man_pages + ./wwctl --emptyconf genconfig man man_pages + cp docs/man/man5/*.5 ./man_pages/ + cd man_pages; for i in wwctl*1 *.5; do gzip --force $$i; echo -n "$$i "; done; echo update_configuration: vendor cmd/update_configuration/update_configuration.go cd cmd/update_configuration && go build \ From 64fd43bb8694578ff668e510309c87d21693e7b7 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 03:52:24 -0600 Subject: [PATCH 04/24] Apply PHONY and remove extraneous comments Several of the targets in Makefile don't create a file of their name, and PHONY identifies these targets to make. Alongside, several comments in Makefile don't provide additional details beyond what is self-documented by the command or target name itself. Signed-off-by: Jonathon Anderson --- Makefile | 60 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 958e700a..8a71dddc 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,21 @@ -.PHONY: all clean contclean - include Variables.mk -# Default target +.PHONY: all all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird -# Validate source and build all packages +.PHONY: build build: lint test-it vet all -# set the go tools into the tools bin. +.PHONY: setup_tools setup_tools: $(GO_TOOLS_BIN) $(GOLANGCI_LINT) -# install go tools into TOOLS_BIN $(GO_TOOLS_BIN): GOBIN="$(PWD)/$(TOOLS_BIN)" go install -mod=vendor $(GO_TOOLS) -# install golangci-lint into TOOLS_BIN $(GOLANGCI_LINT): curl -qq -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) $(GOLANGCI_LINT_VERSION) +.PHONY: setup setup: vendor $(TOOLS_DIR) setup_tools vendor: @@ -30,32 +27,33 @@ endif $(TOOLS_DIR): mkdir -p $@ -# Pre-build steps for source, such as "go generate" config: -# Store configuration for subsequent runs printf " $(foreach V,$(VARLIST),$V := $(strip $($V))\n)" > Defaults.mk - # Global variable search and replace for all *.in files find . -type f -name "*.in" -not -path "./vendor/*" \ -exec sh -c 'sed -ne "$(foreach V,$(VARLIST),s,@$V@,$(strip $($V)),g;)p" $${0} > $${0%.in}' {} \; touch config +.PHONY: rm_config rm_config: rm -f config +.PHONY: genconfig genconfig: rm_config config -# Lint +.PHONY: lint lint: setup_tools $(GOLANGCI_LINT) run --build-tags "$(WW_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... +.PHONY: vet vet: go vet ./... +.PHONY: test-it test-it: go test -v ./... -# Generate test coverage -test-cover: ## Run test coverage and generate html report +.PHONY: test-cover +test-cover: rm -fr coverage mkdir coverage go list -f '{{if gt (len .TestGoFiles) 0}}"go test -covermode count -coverprofile {{.Name}}.coverprofile -coverpkg ./... {{.ImportPath}}"{{end}}' ./... | xargs -I {} bash -c {} @@ -64,8 +62,10 @@ test-cover: ## Run test coverage and generate html report rm *.coverprofile go tool cover -html=coverage/cover.out -o=coverage/cover.html +.PHONY: debian debian: all +.PHONY: files files: all install -d -m 0755 $(DESTDIR)$(BINDIR) install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR) @@ -112,6 +112,7 @@ files: all install -m 0644 staticfiles/x86_64.efi $(DESTDIR)$(WWDATADIR)/ipxe install -m 0644 staticfiles/x86_64.kpxe $(DESTDIR)$(WWDATADIR)/ipxe +.PHONY: init init: systemctl daemon-reload cp -r tftpboot/* $(WWTFTPDIR)/ipxe/ @@ -136,6 +137,7 @@ update_configuration: vendor cmd/update_configuration/update_configuration.go -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf'"\ -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o ../../update_configuration +.PHONY: dist dist: vendor config rm -rf .dist/$(WAREWULF)-$(VERSION) $(WAREWULF)-$(VERSION).tar.gz mkdir -p .dist/$(WAREWULF)-$(VERSION) @@ -143,6 +145,7 @@ dist: vendor config cd .dist; tar -czf ../$(WAREWULF)-$(VERSION).tar.gz $(WAREWULF)-$(VERSION) rm -rf .dist +.PHONY: reference reference: wwctl mkdir -p userdocs/reference ./wwctl --emptyconf genconfig reference userdocs/reference/ @@ -150,16 +153,17 @@ reference: wwctl latexpdf: reference make -C userdocs latexpdf -## wwapi generate code from protobuf. Requires protoc and protoc-grpc-gen-gateway to generate code. -## To setup latest protoc: -## Download the protobuf-all-[VERSION].tar.gz from https://github.com/protocolbuffers/protobuf/releases -## Extract the contents and change in the directory -## ./configure -## make -## make check -## sudo make install -## sudo ldconfig # refresh shared library cache. -## To setup protoc-gen-grpc-gateway, see https://github.com/grpc-ecosystem/grpc-gateway +# wwapi generate code from protobuf. Requires protoc and protoc-grpc-gen-gateway to generate code. +# To setup latest protoc: +# Download the protobuf-all-[VERSION].tar.gz from https://github.com/protocolbuffers/protobuf/releases +# Extract the contents and change in the directory +# ./configure +# make +# make check +# sudo make install +# sudo ldconfig # refresh shared library cache. +# To setup protoc-gen-grpc-gateway, see https://github.com/grpc-ecosystem/grpc-gateway +.PHONY: proto proto: protoc -I /usr/include -I internal/pkg/api/routes/v1 -I=. \ --grpc-gateway_out=. \ @@ -168,15 +172,16 @@ proto: --go-grpc_out=. \ routes.proto -wwapid: ## Build the grpc api server. +wwapid: go build -o ./wwapid internal/app/api/wwapid/wwapid.go -wwapic: ## Build the sample wwapi client. +wwapic: go build -o ./wwapic internal/app/api/wwapic/wwapic.go -wwapird: ## Build the rest api server (revese proxy to the grpc api server). +wwapird: go build -o ./wwapird internal/app/api/wwapird/wwapird.go +.PHONY: contclean contclean: rm -f $(WAREWULF)-$(VERSION).tar.gz rm -f bash_completion @@ -209,9 +214,12 @@ contclean: rm -rf userdocs/_* rm -rf userdocs/reference/* +.PHONY: clean clean: contclean rm -rf vendor +.PHONY: install install: files +.PHONY: debinstall debinstall: files debfiles From c1d56885ac2606f2cf7f265260e25d22b1530365 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:28:49 -0600 Subject: [PATCH 05/24] Refactor Makefile config / Defaults.mk Makefile has included a Defaults.mk file to store variable configuration from one run for implicit inclusion in future runs, but the creation of this file has been automatic and opaque, and the state of it managed by a series of PHONY targets (e.g., config, genconfig, rm_config). This commit refactors config to name its files explicitly (so they can be more properly monitored by make) and makes the generation of Defaults.mk explicit so that its behavior is more obvious. Signed-off-by: Jonathon Anderson --- Dockerfile | 3 +-- Makefile | 25 ++++++++----------- Variables.mk | 3 +++ .../development-environment-vagrant.rst | 2 +- userdocs/quickstart/el8.rst | 2 +- userdocs/quickstart/el9.rst | 2 +- userdocs/quickstart/suse15.rst | 2 +- warewulf.spec.in | 5 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index dfffc58e..616c294d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,7 @@ RUN zypper -n install --no-recommends git go1.18 libgpgme-devel &&\ COPY . /warewulf-src RUN cd /warewulf-src &&\ - make contclean &&\ - make genconfig \ + make contclean Defaults.mk \ PREFIX=/usr \ BINDIRa=/usr/bin \ SYSCONFDIR=/etc \ diff --git a/Makefile b/Makefile index 8a71dddc..667a522f 100644 --- a/Makefile +++ b/Makefile @@ -27,18 +27,16 @@ endif $(TOOLS_DIR): mkdir -p $@ -config: - printf " $(foreach V,$(VARLIST),$V := $(strip $($V))\n)" > Defaults.mk - find . -type f -name "*.in" -not -path "./vendor/*" \ - -exec sh -c 'sed -ne "$(foreach V,$(VARLIST),s,@$V@,$(strip $($V)),g;)p" $${0} > $${0%.in}' {} \; - touch config +.PHONY: config +config: etc/wwapic.conf \ + etc/wwapid.conf \ + etc/wwapird.conf \ + include/systemd/warewulfd.service \ + internal/pkg/config/buildconfig.go \ + warewulf.spec -.PHONY: rm_config -rm_config: - rm -f config - -.PHONY: genconfig -genconfig: rm_config config +%: %.in + sed -ne "$(foreach V,$(VARLIST),s,@$V@,$(strip $($V)),g;)p" $@.in >$@ .PHONY: lint lint: setup_tools @@ -183,11 +181,10 @@ wwapird: .PHONY: contclean contclean: + rm -f Defaults.mk rm -f $(WAREWULF)-$(VERSION).tar.gz rm -f bash_completion - rm -f config - rm -f config_defaults - rm -f Defaults.mk + rm -f config_defaults rm -f etc/wwapi{c,d,rd}.conf rm -f etc/wwapi{c,d,rd}.config rm -f include/systemd/warewulfd.service diff --git a/Variables.mk b/Variables.mk index 43e28ff6..aa22945a 100644 --- a/Variables.mk +++ b/Variables.mk @@ -97,3 +97,6 @@ export GOPROXY # built tags needed for wwbuild binary WW_GO_BUILD_TAGS := containers_image_openpgp containers_image_ostree + +Defaults.mk: + printf " $(foreach V,$(VARLIST),$V := $(strip $($V))\n)" >Defaults.mk diff --git a/userdocs/contributing/development-environment-vagrant.rst b/userdocs/contributing/development-environment-vagrant.rst index 8b3f0ba4..e437f080 100644 --- a/userdocs/contributing/development-environment-vagrant.rst +++ b/userdocs/contributing/development-environment-vagrant.rst @@ -158,7 +158,7 @@ Vagrantfile git clone https://github.com/hpcng/warewulf.git cd warewulf git checkout v4.4.0 - make genconfig \ + make clean Defaults.mk \ PREFIX=/usr \ BINDIR=/usr/bin \ SYSCONFDIR=/etc \ diff --git a/userdocs/quickstart/el8.rst b/userdocs/quickstart/el8.rst index b2ab3772..4e2e0510 100644 --- a/userdocs/quickstart/el8.rst +++ b/userdocs/quickstart/el8.rst @@ -13,7 +13,7 @@ Install Warewulf and dependencies git clone https://github.com/hpcng/warewulf.git cd warewulf - make genconfig \ + make clean Defaults.mk \ PREFIX=/usr \ BINDIR=/usr/bin \ SYSCONFDIR=/etc \ diff --git a/userdocs/quickstart/el9.rst b/userdocs/quickstart/el9.rst index 575dc2b3..0ad66d77 100644 --- a/userdocs/quickstart/el9.rst +++ b/userdocs/quickstart/el9.rst @@ -14,7 +14,7 @@ Install Warewulf and dependencies git clone https://github.com/hpcng/warewulf.git cd warewulf - make genconfig \ + make clean Defaults.mk \ PREFIX=/usr \ BINDIR=/usr/bin \ SYSCONFDIR=/etc \ diff --git a/userdocs/quickstart/suse15.rst b/userdocs/quickstart/suse15.rst index 7c0c58e2..ad6f8786 100644 --- a/userdocs/quickstart/suse15.rst +++ b/userdocs/quickstart/suse15.rst @@ -16,7 +16,7 @@ Install Warewulf and dependencies git clone https://github.com/hpcng/warewulf.git cd warewulf - PREFIX=/usr SYSCONFDIR=/etc TFTPDIR=/srv/tftproot LOCALSTATEDIR=/var/lib make genconfig + PREFIX=/usr SYSCONFDIR=/etc TFTPDIR=/srv/tftproot LOCALSTATEDIR=/var/lib make clean Defaults.mk make all sudo make install diff --git a/warewulf.spec.in b/warewulf.spec.in index f0208f83..8a6b15bc 100644 --- a/warewulf.spec.in +++ b/warewulf.spec.in @@ -71,7 +71,7 @@ system for large clusters of bare metal and/or virtual systems. %build %{?with_offline:OFFLINE_BUILD=1} # Install to sharedstatedir by redirecting LOCALSTATEDIR -make genconfig \ +make Defaults.mk \ PREFIX=%{_prefix} \ BINDIR=%{_bindir} \ SYSCONFDIR=%{_sysconfdir} \ @@ -92,7 +92,8 @@ make %install export NO_BRP_STALE_LINK_ERROR=yes -make install DESTDIR=%{buildroot} +make install \ + DESTDIR=%{buildroot} %pre From 3ef88f8fb0ddc9d4f54b6590aa2754f1cf04fb25 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:30:24 -0600 Subject: [PATCH 06/24] Collapse files target into install Signed-off-by: Jonathon Anderson --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 667a522f..bcd62210 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,8 @@ test-cover: .PHONY: debian debian: all -.PHONY: files -files: all +.PHONY: install +install: all install -d -m 0755 $(DESTDIR)$(BINDIR) install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR) install -d -m 0755 $(DESTDIR)$(WWPROVISIONDIR) @@ -215,8 +215,5 @@ contclean: clean: contclean rm -rf vendor -.PHONY: install -install: files - .PHONY: debinstall debinstall: files debfiles From 88001ecfb0f40f9449831e3644766764d12b5381 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:30:45 -0600 Subject: [PATCH 07/24] Remove extraneous whitespace Signed-off-by: Jonathon Anderson --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index bcd62210..a6571d37 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ setup: vendor $(TOOLS_DIR) setup_tools vendor: ifndef OFFLINE_BUILD - go mod tidy -v - go mod vendor + go mod tidy -v + go mod vendor endif $(TOOLS_DIR): @@ -126,7 +126,7 @@ wwclient: config vendor $(WWCLIENT_DEPS) man_pages: wwctl install -d man_pages - ./wwctl --emptyconf genconfig man man_pages + ./wwctl --emptyconf genconfig man man_pages cp docs/man/man5/*.5 ./man_pages/ cd man_pages; for i in wwctl*1 *.5; do gzip --force $$i; echo -n "$$i "; done; echo @@ -162,7 +162,7 @@ latexpdf: reference # sudo ldconfig # refresh shared library cache. # To setup protoc-gen-grpc-gateway, see https://github.com/grpc-ecosystem/grpc-gateway .PHONY: proto -proto: +proto: protoc -I /usr/include -I internal/pkg/api/routes/v1 -I=. \ --grpc-gateway_out=. \ --grpc-gateway_opt logtostderr=true \ @@ -190,14 +190,14 @@ contclean: rm -f include/systemd/warewulfd.service rm -f internal/pkg/buildconfig/setconfigs.go rm -f internal/pkg/config/buildconfig.go - rm -f man_page - rm -f print_defaults + rm -f man_page + rm -f print_defaults rm -f update_configuration rm -f usr/share/man/man1/ rm -f warewulf.spec rm -f warewulf-*.tar.gz - rm -f wwapic - rm -f wwapid + rm -f wwapic + rm -f wwapid rm -f wwapird rm -f wwclient rm -f wwctl From f375ff32cb1798acc22f33a083b8efcc405ce64b Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:30:57 -0600 Subject: [PATCH 08/24] Remove unused debian targets from Makefile Signed-off-by: Jonathon Anderson --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index a6571d37..e22d21fe 100644 --- a/Makefile +++ b/Makefile @@ -60,9 +60,6 @@ test-cover: rm *.coverprofile go tool cover -html=coverage/cover.out -o=coverage/cover.html -.PHONY: debian -debian: all - .PHONY: install install: all install -d -m 0755 $(DESTDIR)$(BINDIR) @@ -214,6 +211,3 @@ contclean: .PHONY: clean clean: contclean rm -rf vendor - -.PHONY: debinstall -debinstall: files debfiles From 1fdd3742c345ac283cd6c6822b4601cd325b4d1b Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 06:56:45 -0600 Subject: [PATCH 09/24] Improve the generation of man pages Signed-off-by: Jonathon Anderson --- .gitignore | 5 ++--- Makefile | 18 ++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 7859c0c0..2510fccf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ /wwclient /wwctl /bash_completion -/man_page /config_defaults /update_configuration /wwapic @@ -19,9 +18,9 @@ /print_defaults # other created files -/man_pages +/docs/man/man1/*.1.gz +/docs/man/man5/*.5.gz /bash_completion.d -/usr/share/man/man1/ /etc/bash_completion.d/ warewulf.spec internal/pkg/buildconfig/setconfigs.go diff --git a/Makefile b/Makefile index e22d21fe..d5eab3c0 100644 --- a/Makefile +++ b/Makefile @@ -100,8 +100,8 @@ install: all install -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR) install -m 0644 LICENSE.md $(DESTDIR)$(WWDOCDIR) ./wwctl --warewulfconf etc/warewulf.conf genconfig completions > $(DESTDIR)$(BASHCOMPDIR)/wwctl - cp man_pages/*.1* $(DESTDIR)$(MANDIR)/man1/ - cp man_pages/*.5* $(DESTDIR)$(MANDIR)/man5/ + for f in docs/man/man1/*.1.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man1/; done + for f in docs/man/man5/*.5.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man5/; done install -m 0644 staticfiles/README-ipxe.md $(DESTDIR)$(WWDATADIR)/ipxe install -m 0644 staticfiles/arm64.efi $(DESTDIR)$(WWDATADIR)/ipxe install -m 0644 staticfiles/x86_64.efi $(DESTDIR)$(WWDATADIR)/ipxe @@ -121,11 +121,12 @@ wwclient: config vendor $(WWCLIENT_DEPS) cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static" \ -o ../../wwclient -man_pages: wwctl - install -d man_pages - ./wwctl --emptyconf genconfig man man_pages - cp docs/man/man5/*.5 ./man_pages/ - cd man_pages; for i in wwctl*1 *.5; do gzip --force $$i; echo -n "$$i "; done; echo +.PHONY: man_pages +man_pages: wwctl $(wildcard docs/man/man5/*.5) + mkdir -p docs/man/man1 + ./wwctl --emptyconf genconfig man docs/man/man1 + gzip --force docs/man/man1/*.1 + gzip --force --keep docs/man/man5/*.5 update_configuration: vendor cmd/update_configuration/update_configuration.go cd cmd/update_configuration && go build \ @@ -187,10 +188,8 @@ contclean: rm -f include/systemd/warewulfd.service rm -f internal/pkg/buildconfig/setconfigs.go rm -f internal/pkg/config/buildconfig.go - rm -f man_page rm -f print_defaults rm -f update_configuration - rm -f usr/share/man/man1/ rm -f warewulf.spec rm -f warewulf-*.tar.gz rm -f wwapic @@ -204,7 +203,6 @@ contclean: rm -rf .dist/ rm -rf _dist/ rm -rf etc/bash_completion.d/ - rm -rf man_pages rm -rf userdocs/_* rm -rf userdocs/reference/* From 7e31e45a0f57ecf7a47ac8eed07142a5f0c2f065 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 06:57:58 -0600 Subject: [PATCH 10/24] Ignore extracted dists When testing manual mock builds, built dists are extracted in the current directory. This commit ignores those extracted dists so they are not erroneously committed to the repository. Signed-off-by: Jonathon Anderson --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2510fccf..315c804c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ Defaults.mk .dist/ userdocs/_* userdocs/reference/* +/warewulf-?.?.?/ From ec15f0fe362c3128722340b2ae6fbd7a215b2ee8 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 07:18:36 -0600 Subject: [PATCH 11/24] Generate defaults.conf explicitly in Makefile Signed-off-by: Jonathon Anderson --- .gitignore | 1 + Makefile | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 315c804c..51b19d86 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ Defaults.mk /etc/wwapid.conf /etc/wwapic.conf /etc/wwapird.conf +/etc/defaults.conf .dist/ userdocs/_* userdocs/reference/* diff --git a/Makefile b/Makefile index d5eab3c0..b4d6f86d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ include Variables.mk .PHONY: all -all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird +all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird etc/defaults.conf .PHONY: build build: lint test-it vet all @@ -35,6 +35,9 @@ config: etc/wwapic.conf \ internal/pkg/config/buildconfig.go \ warewulf.spec +etc/defaults.conf: wwctl + ./wwctl --emptyconf genconfig defaults >etc/defaults.conf + %: %.in sed -ne "$(foreach V,$(VARLIST),s,@$V@,$(strip $($V)),g;)p" $@.in >$@ @@ -79,7 +82,7 @@ install: all test -f $(DESTDIR)$(WWCONFIGDIR)/wwapic.conf || install -m 644 etc/wwapic.conf $(DESTDIR)$(WWCONFIGDIR) test -f $(DESTDIR)$(WWCONFIGDIR)/wwapid.conf || install -m 644 etc/wwapid.conf $(DESTDIR)$(WWCONFIGDIR) test -f $(DESTDIR)$(WWCONFIGDIR)/wwapird.conf || install -m 644 etc/wwapird.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(DATADIR)/warewulf/defaults.conf || ./wwctl --emptyconf genconfig defaults > $(DESTDIR)$(DATADIR)/warewulf/defaults.conf + test -f $(DESTDIR)$(DATADIR)/warewulf/defaults.conf || install -m 644 etc/defaults.conf $(DESTDIR)$(DATADIR)/warewulf/defaults.conf cp -r etc/examples $(DESTDIR)$(WWCONFIGDIR)/ cp -r etc/ipxe $(DESTDIR)$(WWCONFIGDIR)/ cp -r overlays/* $(DESTDIR)$(WWOVERLAYDIR)/ From bf3995d6f5bd1678b8476726e3e96e90f1bb7ae9 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 07:24:35 -0600 Subject: [PATCH 12/24] Clean up test targets in Makefile Most notably, replaces the test-it target with a more simply named test target. Signed-off-by: Jonathon Anderson --- .github/workflows/test.yml | 2 +- Makefile | 14 +++++++------- userdocs/contributing/debugging.rst | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc829b3d..e582f30d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,4 +41,4 @@ jobs: run: make vet - name: test - run: make test-it + run: make test diff --git a/Makefile b/Makefile index b4d6f86d..d336a82c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ include Variables.mk all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird etc/defaults.conf .PHONY: build -build: lint test-it vet all +build: lint test vet all .PHONY: setup_tools setup_tools: $(GO_TOOLS_BIN) $(GOLANGCI_LINT) @@ -49,17 +49,17 @@ lint: setup_tools vet: go vet ./... -.PHONY: test-it -test-it: - go test -v ./... +.PHONY: test +test: + go test ./... .PHONY: test-cover test-cover: - rm -fr coverage + rm -rf coverage mkdir coverage go list -f '{{if gt (len .TestGoFiles) 0}}"go test -covermode count -coverprofile {{.Name}}.coverprofile -coverpkg ./... {{.ImportPath}}"{{end}}' ./... | xargs -I {} bash -c {} - echo "mode: count" > coverage/cover.out - grep -h -v "^mode:" *.coverprofile >> "coverage/cover.out" + echo "mode: count" >coverage/cover.out + grep -h -v "^mode:" *.coverprofile >>"coverage/cover.out" rm *.coverprofile go tool cover -html=coverage/cover.out -o=coverage/cover.html diff --git a/userdocs/contributing/debugging.rst b/userdocs/contributing/debugging.rst index cf1c256d..d32affe0 100644 --- a/userdocs/contributing/debugging.rst +++ b/userdocs/contributing/debugging.rst @@ -21,12 +21,12 @@ vet`` on the full codebase. Running the full test suite =========================== -The Warewulf ``Makefile`` includes a ``test-it`` target which runs the +The Warewulf ``Makefile`` includes a ``test`` target which runs the full test suite. .. code-block:: console - $ make test-it + $ make test Using delve From 081ead8c9b84b0ec52aad2dedc09c519912aed3f Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:04:52 -0600 Subject: [PATCH 13/24] Use zero-led octals for all install permissions Signed-off-by: Jonathon Anderson --- Makefile | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index d336a82c..09407ec5 100644 --- a/Makefile +++ b/Makefile @@ -77,23 +77,23 @@ install: all install -d -m 0755 $(DESTDIR)$(FIREWALLDDIR) install -d -m 0755 $(DESTDIR)$(SYSTEMDDIR) install -d -m 0755 $(DESTDIR)$(WWDATADIR)/ipxe - test -f $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf || install -m 644 etc/warewulf.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(WWCONFIGDIR)/nodes.conf || install -m 644 etc/nodes.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(WWCONFIGDIR)/wwapic.conf || install -m 644 etc/wwapic.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(WWCONFIGDIR)/wwapid.conf || install -m 644 etc/wwapid.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(WWCONFIGDIR)/wwapird.conf || install -m 644 etc/wwapird.conf $(DESTDIR)$(WWCONFIGDIR) - test -f $(DESTDIR)$(DATADIR)/warewulf/defaults.conf || install -m 644 etc/defaults.conf $(DESTDIR)$(DATADIR)/warewulf/defaults.conf + test -f $(DESTDIR)$(WWCONFIGDIR)/warewulf.conf || install -m 0644 etc/warewulf.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/nodes.conf || install -m 0644 etc/nodes.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/wwapic.conf || install -m 0644 etc/wwapic.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/wwapid.conf || install -m 0644 etc/wwapid.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(WWCONFIGDIR)/wwapird.conf || install -m 0644 etc/wwapird.conf $(DESTDIR)$(WWCONFIGDIR) + test -f $(DESTDIR)$(DATADIR)/warewulf/defaults.conf || install -m 0644 etc/defaults.conf $(DESTDIR)$(DATADIR)/warewulf/defaults.conf cp -r etc/examples $(DESTDIR)$(WWCONFIGDIR)/ cp -r etc/ipxe $(DESTDIR)$(WWCONFIGDIR)/ cp -r overlays/* $(DESTDIR)$(WWOVERLAYDIR)/ - chmod 755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/init + chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/init find $(DESTDIR)$(WWOVERLAYDIR) -type f -name "*.in" -exec rm -f {} \; - chmod 755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwinit - chmod 600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh* - chmod 600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/NetworkManager/system-connections/ww4-managed.ww - chmod 644 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh*.pub.ww - chmod 600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/warewulf/config.ww - chmod 750 $(DESTDIR)$(WWOVERLAYDIR)/host + chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwinit + chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh* + chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/NetworkManager/system-connections/ww4-managed.ww + chmod 0644 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh*.pub.ww + chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/warewulf/config.ww + chmod 0750 $(DESTDIR)$(WWOVERLAYDIR)/host install -m 0755 wwctl $(DESTDIR)$(BINDIR) install -m 0755 wwclient $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwclient install -m 0755 wwapic $(DESTDIR)$(BINDIR) From 1c3b566d43f2d2f7c72d3451d5e8635bfc764032 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:05:58 -0600 Subject: [PATCH 14/24] Test targets depend on config Signed-off-by: Jonathon Anderson --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 09407ec5..47a91a37 100644 --- a/Makefile +++ b/Makefile @@ -46,15 +46,15 @@ lint: setup_tools $(GOLANGCI_LINT) run --build-tags "$(WW_GO_BUILD_TAGS)" --skip-dirs internal/pkg/staticfiles ./... .PHONY: vet -vet: +vet: config go vet ./... .PHONY: test -test: +test: config go test ./... .PHONY: test-cover -test-cover: +test-cover: config rm -rf coverage mkdir coverage go list -f '{{if gt (len .TestGoFiles) 0}}"go test -covermode count -coverprofile {{.Name}}.coverprofile -coverpkg ./... {{.ImportPath}}"{{end}}' ./... | xargs -I {} bash -c {} From daab6075d7144856befd2c1fcad3c1eddcd474fa Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 08:02:24 -0600 Subject: [PATCH 15/24] Remove extraneous rm from clean Makefile target Signed-off-by: Jonathon Anderson --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 47a91a37..bc2b4ab0 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,6 @@ install: all cp -r etc/ipxe $(DESTDIR)$(WWCONFIGDIR)/ cp -r overlays/* $(DESTDIR)$(WWOVERLAYDIR)/ chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/init - find $(DESTDIR)$(WWOVERLAYDIR) -type f -name "*.in" -exec rm -f {} \; chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwinit chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh* chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/NetworkManager/system-connections/ww4-managed.ww From 0b9702a7d340b6165ffbbf40fe036ecc48a8f790 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:08:56 -0600 Subject: [PATCH 16/24] Separate out bash completions in Makefile Signed-off-by: Jonathon Anderson --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bc2b4ab0..d80b75b4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ include Variables.mk .PHONY: all -all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird etc/defaults.conf +all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird etc/defaults.conf etc/bash_completion.d/wwctl .PHONY: build build: lint test vet all @@ -101,7 +101,7 @@ install: all install -m 0644 include/firewalld/warewulf.xml $(DESTDIR)$(FIREWALLDDIR) install -m 0644 include/systemd/warewulfd.service $(DESTDIR)$(SYSTEMDDIR) install -m 0644 LICENSE.md $(DESTDIR)$(WWDOCDIR) - ./wwctl --warewulfconf etc/warewulf.conf genconfig completions > $(DESTDIR)$(BASHCOMPDIR)/wwctl + install -m 0644 etc/bash_completion.d/wwctl $(DESTDIR)$(BASHCOMPDIR)/wwctl for f in docs/man/man1/*.1.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man1/; done for f in docs/man/man5/*.5.gz; do install -m 0644 $$f $(DESTDIR)$(MANDIR)/man5/; done install -m 0644 staticfiles/README-ipxe.md $(DESTDIR)$(WWDATADIR)/ipxe @@ -109,6 +109,10 @@ install: all install -m 0644 staticfiles/x86_64.efi $(DESTDIR)$(WWDATADIR)/ipxe install -m 0644 staticfiles/x86_64.kpxe $(DESTDIR)$(WWDATADIR)/ipxe +etc/bash_completion.d/wwctl: wwctl + mkdir -p etc/bash_completion.d/ + ./wwctl --warewulfconf etc/warewulf.conf genconfig completions >etc/bash_completion.d/wwctl + .PHONY: init init: systemctl daemon-reload From 5ff899293e2022d239e073cca81bec63cb14a4f3 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:11:11 -0600 Subject: [PATCH 17/24] Manage examples, ipxe, and overlays with install Signed-off-by: Jonathon Anderson --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d80b75b4..81212ff2 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ install: all install -d -m 0755 $(DESTDIR)$(WWCHROOTDIR) install -d -m 0755 $(DESTDIR)$(WWPROVISIONDIR) install -d -m 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR) + install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/examples install -d -m 0755 $(DESTDIR)$(WWCONFIGDIR)/ipxe install -d -m 0755 $(DESTDIR)$(BASHCOMPDIR) install -d -m 0755 $(DESTDIR)$(MANDIR)/man1 @@ -83,9 +84,9 @@ install: all test -f $(DESTDIR)$(WWCONFIGDIR)/wwapid.conf || install -m 0644 etc/wwapid.conf $(DESTDIR)$(WWCONFIGDIR) test -f $(DESTDIR)$(WWCONFIGDIR)/wwapird.conf || install -m 0644 etc/wwapird.conf $(DESTDIR)$(WWCONFIGDIR) test -f $(DESTDIR)$(DATADIR)/warewulf/defaults.conf || install -m 0644 etc/defaults.conf $(DESTDIR)$(DATADIR)/warewulf/defaults.conf - cp -r etc/examples $(DESTDIR)$(WWCONFIGDIR)/ - cp -r etc/ipxe $(DESTDIR)$(WWCONFIGDIR)/ - cp -r overlays/* $(DESTDIR)$(WWOVERLAYDIR)/ + for f in etc/examples/*.ww; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/examples/; done + for f in etc/ipxe/*.ipxe; do install -m 0644 $$f $(DESTDIR)$(WWCONFIGDIR)/ipxe/; done + (cd overlays && find * -type f -exec install -D -m 0644 {} $(DESTDIR)$(WWOVERLAYDIR)/{} \;) chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/init chmod 0755 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/$(WWCLIENTDIR)/wwinit chmod 0600 $(DESTDIR)$(WWOVERLAYDIR)/wwinit/etc/ssh/ssh* From 0561c3009ac0be047ba2a33d8110d2e648d29ce6 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 09:11:53 -0600 Subject: [PATCH 18/24] Improve specification of command builds Signed-off-by: Jonathon Anderson --- Makefile | 35 ++++++++++++++++------------------- Variables.mk | 2 -- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 81212ff2..6ed02448 100644 --- a/Makefile +++ b/Makefile @@ -120,13 +120,11 @@ init: cp -r tftpboot/* $(WWTFTPDIR)/ipxe/ restorecon -r $(WWTFTPDIR) -wwctl: config vendor $(WWCTL_DEPS) - cd cmd/wwctl; GOOS=linux go build -mod vendor -tags "$(WW_GO_BUILD_TAGS)" \ - -o ../../wwctl +wwctl: config vendor $(call godeps,cmd/wwctl/main.go) + GOOS=linux go build -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o wwctl cmd/wwctl/main.go -wwclient: config vendor $(WWCLIENT_DEPS) - cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static" \ - -o ../../wwclient +wwclient: config vendor $(call godeps,cmd/wwclient/main.go) + CGO_ENABLED=0 GOOS=linux go build -mod vendor -a -ldflags "-extldflags -static" -o wwclient cmd/wwclient/main.go .PHONY: man_pages man_pages: wwctl $(wildcard docs/man/man5/*.5) @@ -135,10 +133,18 @@ man_pages: wwctl $(wildcard docs/man/man5/*.5) gzip --force docs/man/man1/*.1 gzip --force --keep docs/man/man5/*.5 -update_configuration: vendor cmd/update_configuration/update_configuration.go - cd cmd/update_configuration && go build \ - -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf'"\ - -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o ../../update_configuration +update_configuration: vendor $(call godeps,cmd/update_configuration/update_configuration.go) + go build -X 'github.com/hpcng/warewulf/internal/pkg/node.ConfigFile=./etc/nodes.conf'" \ + -mod vendor -tags "$(WW_GO_BUILD_TAGS)" -o update_configuration cmd/update_configuration/update_configuration.go + +wwapid: $(call godeps,internal/app/api/wwapid/wwapid.go) + go build -o ./wwapid internal/app/api/wwapid/wwapid.go + +wwapic: $(call godeps,internal/app/api/wwapic/wwapic.go) + go build -o ./wwapic internal/app/api/wwapic/wwapic.go + +wwapird: $(call godeps,internal/app/api/wwapird/wwapird.go) + go build -o ./wwapird internal/app/api/wwapird/wwapird.go .PHONY: dist dist: vendor config @@ -175,15 +181,6 @@ proto: --go-grpc_out=. \ routes.proto -wwapid: - go build -o ./wwapid internal/app/api/wwapid/wwapid.go - -wwapic: - go build -o ./wwapic internal/app/api/wwapic/wwapic.go - -wwapird: - go build -o ./wwapird internal/app/api/wwapird/wwapird.go - .PHONY: contclean contclean: rm -f Defaults.mk diff --git a/Variables.mk b/Variables.mk index aa22945a..7195d0d3 100644 --- a/Variables.mk +++ b/Variables.mk @@ -88,8 +88,6 @@ GOLANGCI_LINT_VERSION := v1.53.2 # helper functions godeps=$(shell go list -deps -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}' $(1) | sed "s%${PWD}/%%g") -WWCTL_DEPS:=$(call godeps,cmd/wwctl/main.go) -WWCLIENT_DEPS:=$(call godeps,cmd/wwclient/main.go) # use GOPROXY for older git clients and speed up downloads GOPROXY ?= https://proxy.golang.org From d2b63ecb750fae1afd0a003ccf0f548a475d391f Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:16:03 -0600 Subject: [PATCH 19/24] Simplier dist generation prep Signed-off-by: Jonathon Anderson --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ed02448..11e577b4 100644 --- a/Makefile +++ b/Makefile @@ -148,7 +148,7 @@ wwapird: $(call godeps,internal/app/api/wwapird/wwapird.go) .PHONY: dist dist: vendor config - rm -rf .dist/$(WAREWULF)-$(VERSION) $(WAREWULF)-$(VERSION).tar.gz + rm -rf .dist/ $(WAREWULF)-$(VERSION).tar.gz mkdir -p .dist/$(WAREWULF)-$(VERSION) rsync -a --exclude=".*" --exclude "*~" * .dist/$(WAREWULF)-$(VERSION)/ cd .dist; tar -czf ../$(WAREWULF)-$(VERSION).tar.gz $(WAREWULF)-$(VERSION) From 9ea3381b9d4a1658fbdefcc9720e5a25810aa05a Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:16:18 -0600 Subject: [PATCH 20/24] Remove extraneous backup file Signed-off-by: Jonathon Anderson --- .../vers42/nodes.conf.bak | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 cmd/update_configuration/vers42/nodes.conf.bak diff --git a/cmd/update_configuration/vers42/nodes.conf.bak b/cmd/update_configuration/vers42/nodes.conf.bak deleted file mode 100644 index 792e48a2..00000000 --- a/cmd/update_configuration/vers42/nodes.conf.bak +++ /dev/null @@ -1,22 +0,0 @@ -nodeprofiles: - default: - comment: This profile is automatically included for each node - runtime overlay: "generic" - discoverable: false - leap: - comment: openSUSE leap - kernel version: "5.14.21" - ipmi netmask: "255.255.255.0" - keys: - foo: baar - network devices: - lan1: - gateway: 1.1.1.1 -nodes: - node01: - system overlay: "nodeoverlay" - discoverable: true - network devices: - eth0: - ipaddr: 1.2.3.4 - default: true From b3b759179647f9fc560881e0ec4c114c979bfa74 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 09:42:18 -0600 Subject: [PATCH 21/24] Audit Makefile clean target and gitignore Signed-off-by: Jonathon Anderson --- .gitignore | 14 +------------- Makefile | 15 ++------------- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 51b19d86..55581c1c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,34 +5,22 @@ /vendor # binaries -/warewulfd -/wwbuild /wwclient /wwctl -/bash_completion -/config_defaults /update_configuration /wwapic /wwapid /wwapird -/print_defaults # other created files -/docs/man/man1/*.1.gz +/docs/man/man1 /docs/man/man5/*.5.gz -/bash_completion.d /etc/bash_completion.d/ warewulf.spec -internal/pkg/buildconfig/setconfigs.go internal/pkg/config/buildconfig.go include/systemd/warewulfd.service -_dist/ -/config warewulf-*.tar.gz Defaults.mk -/etc/wwapid.config -/etc/wwapic.config -/etc/wwapird.config /etc/wwapid.conf /etc/wwapic.conf /etc/wwapird.conf diff --git a/Makefile b/Makefile index 11e577b4..6a65a2df 100644 --- a/Makefile +++ b/Makefile @@ -185,30 +185,19 @@ proto: contclean: rm -f Defaults.mk rm -f $(WAREWULF)-$(VERSION).tar.gz - rm -f bash_completion - rm -f config_defaults - rm -f etc/wwapi{c,d,rd}.conf - rm -f etc/wwapi{c,d,rd}.config + rm -f wwapi{c,d,rd} etc/wwapi{c,d,rd}.conf rm -f include/systemd/warewulfd.service - rm -f internal/pkg/buildconfig/setconfigs.go rm -f internal/pkg/config/buildconfig.go - rm -f print_defaults rm -f update_configuration rm -f warewulf.spec - rm -f warewulf-*.tar.gz - rm -f wwapic - rm -f wwapid - rm -f wwapird rm -f wwclient rm -f wwctl rm -rf $(TOOLS_DIR) - rm -rf bash_completion.d - rm -rf /config rm -rf .dist/ - rm -rf _dist/ rm -rf etc/bash_completion.d/ rm -rf userdocs/_* rm -rf userdocs/reference/* + rm -rf etc/defaults.conf .PHONY: clean clean: contclean From b6c774c4b56e11ec46f6c6d03cee95e1bfdcd0a7 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:20:52 -0600 Subject: [PATCH 22/24] Use no config file when generating completions Signed-off-by: Jonathon Anderson --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a65a2df..7b763551 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ install: all etc/bash_completion.d/wwctl: wwctl mkdir -p etc/bash_completion.d/ - ./wwctl --warewulfconf etc/warewulf.conf genconfig completions >etc/bash_completion.d/wwctl + ./wwctl --emptyconf genconfig completions >etc/bash_completion.d/wwctl .PHONY: init init: From 526c96fb5ec519fa37ba2bf12156ae210bfa9322 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Thu, 10 Aug 2023 11:45:10 -0600 Subject: [PATCH 23/24] Update changelog to reflect Makefile changes Signed-off-by: Jonathon Anderson --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df7cede1..e9530dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Check for formal correct IP and MAC addresses for command line options and when reading in the configurations - Write log messages to stderr rather than stdout. #768 +- Updates to Makefile for clarity, notably removing genconfig and replacing + test-it with test. #890 ## [4.4.0] 2023-01-18 From 9a453c17022f7c3318226051f01d0e69bcd572d9 Mon Sep 17 00:00:00 2001 From: Jonathon Anderson Date: Fri, 11 Aug 2023 16:15:33 -0600 Subject: [PATCH 24/24] Capture proto dependency installs as targets Signed-off-by: Jonathon Anderson --- Makefile | 29 ++++++++++++++++------------- Variables.mk | 8 ++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 7b763551..3b51f42a 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ all: config vendor wwctl wwclient man_pages wwapid wwapic wwapird etc/defaults.c build: lint test vet all .PHONY: setup_tools -setup_tools: $(GO_TOOLS_BIN) $(GOLANGCI_LINT) +setup_tools: $(GO_TOOLS_BIN) $(GOLANGCI_LINT) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(GO_TOOLS_BIN): GOBIN="$(PWD)/$(TOOLS_BIN)" go install -mod=vendor $(GO_TOOLS) @@ -15,6 +15,19 @@ $(GO_TOOLS_BIN): $(GOLANGCI_LINT): curl -qq -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TOOLS_BIN) $(GOLANGCI_LINT_VERSION) +$(PROTOC): + cd $(PWD)/$(TOOLS_DIR) && curl -LO $(PROTOC_URL) && unzip protoc-24.0-linux-aarch_64.zip + +$(PROTOC_GEN_GRPC_GATEWAY): + curl -L $(PROTOC_GEN_GRPC_GATEWAY_URL) -o $(PROTOC_GEN_GRPC_GATEWAY) + chmod +x $(PROTOC_GEN_GRPC_GATEWAY) + +$(PROTOC_GEN_GO): + GOBIN="$(PWD)/$(TOOLS_BIN)" go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + +$(PROTOC_GEN_GO_GRPC): + GOBIN="$(PWD)/$(TOOLS_BIN)" go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + .PHONY: setup setup: vendor $(TOOLS_DIR) setup_tools @@ -162,19 +175,9 @@ reference: wwctl latexpdf: reference make -C userdocs latexpdf -# wwapi generate code from protobuf. Requires protoc and protoc-grpc-gen-gateway to generate code. -# To setup latest protoc: -# Download the protobuf-all-[VERSION].tar.gz from https://github.com/protocolbuffers/protobuf/releases -# Extract the contents and change in the directory -# ./configure -# make -# make check -# sudo make install -# sudo ldconfig # refresh shared library cache. -# To setup protoc-gen-grpc-gateway, see https://github.com/grpc-ecosystem/grpc-gateway .PHONY: proto -proto: - protoc -I /usr/include -I internal/pkg/api/routes/v1 -I=. \ +proto: $(PROTOC) $(PROTOC_GEN_GRPC_GATEWAY) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) + PATH=$(TOOLS_BIN):$(PATH) $(PROTOC) -I /usr/include -I internal/pkg/api/routes/v1 -I=. \ --grpc-gateway_out=. \ --grpc-gateway_opt logtostderr=true \ --go_out=. \ diff --git a/Variables.mk b/Variables.mk index 7195d0d3..9e650460 100644 --- a/Variables.mk +++ b/Variables.mk @@ -85,6 +85,14 @@ GO_TOOLS_BIN := $(addprefix $(TOOLS_BIN)/, $(notdir $(GO_TOOLS))) GO_TOOLS_VENDOR := $(addprefix vendor/, $(GO_TOOLS)) GOLANGCI_LINT := $(TOOLS_BIN)/golangci-lint GOLANGCI_LINT_VERSION := v1.53.2 +PROTOC_GEN_GO := $(TOOLS_BIN)/protoc-gen-go +PROTOC_GEN_GO_GRPC := $(TOOLS_BIN)/protoc-gen-go-grpc +PROTOC := $(TOOLS_BIN)/protoc +PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protoc-24.0-linux-x86_64.zip +#PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v24.0/protoc-24.0-linux-aarch_64.zip +PROTOC_GEN_GRPC_GATEWAY := $(TOOLS_BIN)/protoc-gen-grpc-gateway +PROTOC_GEN_GRPC_GATEWAY_URL := https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.16.2/protoc-gen-grpc-gateway-v2.16.2-linux-x86_64 +#PROTOC_GEN_GRPC_GATEWAY_URL := https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v2.16.2/protoc-gen-grpc-gateway-v2.16.2-linux-arm64 # helper functions godeps=$(shell go list -deps -f '{{if not .Standard}}{{ $$dep := . }}{{range .GoFiles}}{{$$dep.Dir}}/{{.}} {{end}}{{end}}' $(1) | sed "s%${PWD}/%%g")