Lots of code cleanup, better error handling, etc...

Makefile

@@ -1,6 +1,8 @@
+.PHONY: all
 
+all: warewulfd wwbuild wwclient
 
-files:
+files: all
 	sudo install -d -m 0755 /var/warewulf/provision
 	sudo install -d -m 0755 /var/warewulf/provision/kernels
 	sudo install -d -m 0755 /var/warewulf/provision/overlays
@@ -12,6 +14,8 @@ files:
 	sudo cp -r tftpboot/* /var/lib/tftpboot/warewulf/ipxe/
 	sudo cp -r overlays /etc/warewulf/
 	sudo chmod +x /etc/warewulf/overlays/generic/init
+	sudo mkdir -p /etc/warewulf/overlays/generic/warewulf/bin/
+	sudo cp wwclient /etc/warewulf/overlays/generic/warewulf/bin/
 
 services: files
 	sudo systemctl enable tftp
@@ -19,13 +23,19 @@ services: files
 	sudo systemctl enable dhcpd
 	sudo systemctl restart dhcpd
 
-build:
+warewulfd:
 	cd cmd/warewulfd; go build -o ../../warewulfd
+
+wwbuild:
 	cd cmd/wwbuild; go build -o ../../wwbuild
 
+wwclient:
+	cd cmd/wwclient; CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags -static' -o ../../wwclient
+
 clean:
 	rm -f warewulfd
 	rm -f wwbuild
+	rm -f wwclient
 
-install: build files services
+install: files services
 

cmd/warewulfd/ipxe.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"github.com/hpcng/warewulf/internal/pkg/assets"
+	"log"
+	"net/http"
+	"strings"
+)
+
+func ipxe(w http.ResponseWriter, req *http.Request) {
+	url := strings.Split(req.URL.Path, "/")
+
+	if url[2] == "" {
+		fmt.Printf("ERROR: Bad iPXE request from %s\n", req.RemoteAddr)
+		return
+	}
+
+	hwaddr := strings.ReplaceAll(url[2], "-", ":")
+	node, err := assets.FindByHwaddr(hwaddr)
+	if err != nil {
+		log.Printf("Could not find HW Addr: %s\n", hwaddr)
+		w.WriteHeader(404)
+		return
+	}
+
+	if node.HostName != "" {
+		fmt.Printf("IPXE:  %15s: hwaddr=%s\n", node.Fqdn, hwaddr)
+
+		fmt.Fprintf(w, "#!ipxe\n")
+
+		fmt.Fprintf(w, "echo Now booting Warewulf - v4 Proof of Concept\n")
+		fmt.Fprintf(w, "set base http://192.168.1.1:9873/\n")
+		fmt.Fprintf(w, "kernel ${base}/kernel/%s crashkernel=no quiet\n", url[2])
+		fmt.Fprintf(w, "initrd ${base}/vnfs/%s\n", url[2])
+		fmt.Fprintf(w, "initrd ${base}/kmods/%s\n", url[2])
+		fmt.Fprintf(w, "initrd ${base}/overlay/%s\n", url[2])
+		fmt.Fprintf(w, "boot\n")
+	} else {
+		fmt.Printf("ERROR: iPXE request from unknown Node (hwaddr=%s)\n", url[2])
+	}
+	return
+}

cmd/warewulfd/kernel.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func kernel(w http.ResponseWriter, req *http.Request) {
+
+	node, err := getSanity(req)
+	if err != nil {
+		w.WriteHeader(404)
+		log.Panicln(err)
+		return
+	}
+
+	if node.KernelVersion != "" {
+		fileName := fmt.Sprintf("%s/provision/kernels/vmlinuz-%s", LocalStateDir, node.KernelVersion)
+
+		err := sendFile(w, fileName, node.Fqdn)
+		if err != nil {
+			log.Println(err)
+		}
+
+	} else {
+		w.WriteHeader(503)
+		log.Printf("No kernel version set for node %s\n", node.Fqdn)
+	}
+
+	return
+}

cmd/warewulfd/kmods.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func kmods(w http.ResponseWriter, req *http.Request) {
+
+	node, err := getSanity(req)
+	if err != nil {
+		w.WriteHeader(404)
+		log.Panicln(err)
+		return
+	}
+	if node.KernelVersion != "" {
+		fileName := fmt.Sprintf("%s/provision/kernels/kmods-%s.img", LocalStateDir, node.KernelVersion)
+
+		err := sendFile(w, fileName, node.Fqdn)
+		if err != nil {
+			log.Println(err)
+		}
+
+	} else {
+		w.WriteHeader(503)
+		log.Printf("No kernel version set for node %s\n", node.Fqdn)
+	}
+
+	return
+}

cmd/warewulfd/overlay.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func overlay(w http.ResponseWriter, req *http.Request) {
+
+	node, err := getSanity(req)
+	if err != nil {
+		w.WriteHeader(404)
+		log.Panicln(err)
+		return
+	}
+
+	if node.Overlay!= "" {
+		fileName := fmt.Sprintf("%s/provision/overlays/%s.img", LocalStateDir, node.Fqdn)
+
+		err := sendFile(w, fileName, node.Fqdn)
+		if err != nil {
+			log.Println(err)
+		}
+	} else {
+		w.WriteHeader(503)
+		log.Printf("No Overlay set for node %s\n", node.Fqdn)
+	}
+
+	return
+}

cmd/warewulfd/runtime.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+func runtime(w http.ResponseWriter, req *http.Request) {
+
+	node, err := getSanity(req)
+	if err != nil {
+		w.WriteHeader(404)
+		log.Panicln(err)
+		return
+	}
+
+	remote := strings.Split(req.RemoteAddr, ":")
+	port, err := strconv.Atoi(remote[1])
+	if err != nil {
+		w.WriteHeader(404)
+		log.Printf("Could not convert port to integer: %s\n", remote[1])
+		return
+	}
+
+	if port >= 1024 {
+		log.Panicf("DENIED: Connection coming from non-privledged port: %s\n", req.RemoteAddr)
+		return
+	}
+
+	if node.Overlay!= "" {
+		fileName := fmt.Sprintf("%s/provision/runtime/%s.img", LocalStateDir, node.Fqdn)
+
+		err := sendFile(w, fileName, node.Fqdn)
+		if err != nil {
+			log.Println(err)
+		}
+	} else {
+		w.WriteHeader(503)
+		log.Printf("No Overlay set for node %s\n", node.Fqdn)
+	}
+
+	return
+}

cmd/warewulfd/vnfs.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"path"
+)
+
+func vnfs(w http.ResponseWriter, req *http.Request) {
+
+	node, err := getSanity(req)
+	if err != nil {
+		w.WriteHeader(404)
+		log.Panicln(err)
+		return
+	}
+
+	if node.Vnfs != "" {
+		fileName := fmt.Sprintf("%s/provision/bases/%s.img.gz", LocalStateDir, path.Base(node.Vnfs))
+
+		err := sendFile(w, fileName, node.Fqdn)
+		if err != nil {
+			log.Println(err)
+		}
+	} else {
+		w.WriteHeader(503)
+		log.Printf("No VNFS set for node %s\n", node.Fqdn)
+	}
+
+	return
+}

cmd/warewulfd/warewulfd.go

@@ -5,10 +5,10 @@ import (
     "fmt"
     "io"
     "os"
-    "path"
     "strconv"
     "strings"
 
+    "github.com/hpcng/warewulf/internal/pkg/errors"
     "github.com/hpcng/warewulf/internal/pkg/assets"
     "net/http"
 )
@@ -18,41 +18,33 @@ import (
 
 const LocalStateDir = "/var/warewulf"
 
-func ipxe(w http.ResponseWriter, req *http.Request) {
+func getSanity(req *http.Request) (assets.NodeInfo, error) {
     url := strings.Split(req.URL.Path, "/")
 
-    if url[2] == "" {
-        fmt.Printf("ERROR: Bad iPXE request from %s\n", req.RemoteAddr)
-        return
-    }
-
     hwaddr := strings.ReplaceAll(url[2], "-", ":")
-    node := assets.FindByHwaddr(hwaddr)
-
-    if node.HostName != "" {
-        fmt.Printf("IPXE:  %15s: hwaddr=%s\n", node.Fqdn, hwaddr)
-
-        fmt.Fprintf(w, "#!ipxe\n")
-
-        fmt.Fprintf(w, "echo Now booting Warewulf - v4 Proof of Concept\n")
-        fmt.Fprintf(w, "set base http://192.168.1.1:9873/\n")
-        fmt.Fprintf(w, "kernel ${base}/files/kernel/%s crashkernel=no quiet\n", url[2])
-        fmt.Fprintf(w, "initrd ${base}/files/vnfs/%s\n", url[2])
-        fmt.Fprintf(w, "initrd ${base}/files/kmods/%s\n", url[2])
-        fmt.Fprintf(w, "initrd ${base}/files/overlay/%s\n", url[2])
-        fmt.Fprintf(w, "boot\n")
-    } else {
-        fmt.Printf("ERROR: iPXE request from unknown Node (hwaddr=%s)\n", url[2])
+    node, err := assets.FindByHwaddr(hwaddr)
+    if err != nil {
+        return node, errors.New("Could not find HW address")
     }
-    return
+
+    if node.Fqdn == "" {
+        fmt.Printf("UNKNOWN: %15s: %s\n", node.Fqdn, req.URL.Path)
+        return node, errors.New("Unknown Node: "+ hwaddr)
+    }
+    fmt.Printf("GET:   %15s: %s\n", node.Fqdn, req.URL.Path)
+
+    return node, nil
 }
 
-
+/*
 func files(w http.ResponseWriter, req *http.Request) {
     url := strings.Split(req.URL.Path, "/")
 
     node := assets.FindByHwaddr(strings.ReplaceAll(url[3], "-", ":"))
 
+    if node.Fqdn == "" {
+        fmt.Printf("UNKNOWN: %15s: %s\n", node.Fqdn, req.URL.Path)
+    }
     fmt.Printf("GET:   %15s: %s\n", node.Fqdn, req.URL.Path)
 
     if url[2] == "kernel" {
@@ -77,21 +69,38 @@ func files(w http.ResponseWriter, req *http.Request) {
         if node.Overlay!= "" {
             overlayFile := fmt.Sprintf("%s/provision/overlays/%s.img", LocalStateDir, node.Fqdn)
 
+            sendFile(w, overlayFile, node.Fqdn)
+        }
+    } else if url[2] == "runtime" {
+        fmt.Printf("FROM: %s\n", req.RemoteAddr)
+
+        remote := strings.Split(req.RemoteAddr, ":")
+        port, _ := strconv.Atoi(remote[1])
+
+        if port >= 1024 {
+            fmt.Printf("DENIED: Connection coming from non-privledged port: %s\n", req.RemoteAddr)
+            return
+        }
+
+        if node.Overlay!= "" {
+            overlayFile := fmt.Sprintf("%s/provision/runtime/%s.img", LocalStateDir, node.Fqdn)
+
             sendFile(w, overlayFile, node.Fqdn)
         }
     }
+
     return
 }
+ */
 
-
-func sendFile(w http.ResponseWriter, filename string, sendto string) {
+func sendFile(w http.ResponseWriter, filename string, sendto string) error {
 
     fmt.Printf("SEND:  %15s: %s\n", sendto, filename)
 
     fd, err := os.Open(filename)
     if err != nil {
-        fmt.Println("ERROR:   %s\n", err)
-        return
+        fmt.Printf("ERROR:   %s\n", err)
+        return err
     }
 
     FileHeader := make([]byte, 512)
@@ -106,14 +115,20 @@ func sendFile(w http.ResponseWriter, filename string, sendto string) {
 
     fd.Seek(0, 0)
     io.Copy(w, fd)
-}
 
+    fd.Close()
+    return nil
+}
 
 
 func main() {
 
     http.HandleFunc("/ipxe/", ipxe)
-    http.HandleFunc("/files/", files)
+    http.HandleFunc("/kernel/", kernel)
+    http.HandleFunc("/kmods/", kmods)
+    http.HandleFunc("/vnfs/", vnfs)
+    http.HandleFunc("/overlay/", overlay)
+    http.HandleFunc("/runtime/", runtime)
 
     http.ListenAndServe(":9873", nil)
 }

cmd/wwbuild/wwbuild.go

@@ -5,6 +5,7 @@ import (
     "fmt"
     "github.com/hpcng/warewulf/internal/pkg/assets"
     "github.com/hpcng/warewulf/internal/pkg/util"
+    "log"
     "os"
     "os/exec"
     "path"
@@ -44,12 +45,24 @@ func main(){
         if len(os.Args) >= 3 {
             vnfsBuild(os.Args[3])
         } else {
-            for _, vnfs := range assets.FindAllVnfs() {
+            nodeList, err := assets.FindAllVnfs()
+            if err != nil {
+                log.Panicf("Could not locate VNFS images: %s\n", err)
+                os.Exit(1)
+            }
+
+            for _, vnfs := range nodeList {
                 vnfsBuild(vnfs)
             }
         }
     } else if os.Args[1] == "kernel" {
-        for _, kernelVers := range assets.FindAllKernels() {
+        nodeList, err := assets.FindAllKernels()
+        if err != nil {
+            log.Panicf("Could not locate Kernel Versions: %s\n", err)
+            os.Exit(1)
+        }
+
+        for _, kernelVers := range nodeList {
             kernelSource := fmt.Sprintf("/boot/vmlinuz-%s", kernelVers)
             // TODO: Check time stamps of source and dests to see if we need to rebuild or skip
             if _, err := os.Stat(kernelSource); err == nil {
@@ -74,7 +87,13 @@ func main(){
         }
     } else if os.Args[1] == "overlay" {
         //TODO: Move this all to warewulfd and generate on demand when needed
-        for _, node := range assets.FindAllNodes() {
+        nodeList, err := assets.FindAllNodes()
+        if err != nil {
+            log.Panicf("Could not identify nodes: %s\n", err)
+            os.Exit(1)
+        }
+
+        for _, node := range nodeList {
 
             overlayDir := fmt.Sprintf("/etc/warewulf/overlays/%s", node.Overlay)
 

cmd/wwclient/wwclient.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+)
+
+
+
+
+func main() {
+	localAddr, err := net.ResolveIPAddr("ip", "localhost")
+	if err != nil {
+		panic(err)
+	}
+
+	// You also need to do this to make it work and not give you a
+	// "mismatched local address type ip"
+	// This will make the ResolveIPAddr a TCPAddr without needing to
+	// say what SRC port number to use.
+	localTCPAddr := net.TCPAddr{
+		IP: localAddr.IP,
+		Port: 987,
+	}
+
+	webclient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				LocalAddr: &localTCPAddr,
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+
+
+	for true {
+		var resp *http.Response
+
+		for true {
+			var err error
+			fmt.Printf("Connecting ....\n")
+			resp, err = webclient.Get("http://localhost:9873/files/runtime/xx-xx-xx-xx-xx-xx")
+			if err == nil {
+				break
+			} else {
+				fmt.Println(err)
+			}
+			time.Sleep(1000 * time.Millisecond)
+		}
+
+		fmt.Println("Response status:", resp.Status)
+		scanner := bufio.NewScanner(resp.Body)
+		for i := 0; scanner.Scan() && i < 5; i++ {
+			fmt.Println(scanner.Text())
+		}
+		if err := scanner.Err(); err != nil {
+			panic(err)
+		}
+		resp.Body.Close()
+		time.Sleep(5000 * time.Millisecond)
+	}
+}

internal/pkg/assets/assets.go

@@ -1,13 +1,11 @@
 
 package assets
 
-
-
 import (
-    "fmt"
+    "gopkg.in/yaml.v2"
     "io/ioutil"
 
-    "gopkg.in/yaml.v2"
+    "github.com/hpcng/warewulf/internal/pkg/errors"
 )
 
 
@@ -62,18 +60,18 @@ type NodeInfo struct {
 }
 
 
-func FindAllNodes() []NodeInfo {
+func FindAllNodes() ([]NodeInfo, error) {
     var c nodeYaml
     var ret []NodeInfo
 
     fd, err := ioutil.ReadFile(ConfigFile)
     if err != nil {
-        fmt.Println(err)
+        return nil, err
     }
 
     err = yaml.Unmarshal(fd, &c)
     if err != nil {
-        fmt.Println(err)
+        return nil, err
     }
 
     for groupname, group := range c.NodeGroups {
@@ -110,30 +108,40 @@ func FindAllNodes() []NodeInfo {
         }
     }
 
-    return ret
+    return ret, nil
 }
 
 
-func FindByHwaddr(hwa string) NodeInfo{
+func FindByHwaddr(hwa string) (NodeInfo, error) {
     var ret NodeInfo
 
-    for _, node := range FindAllNodes() {
+    nodeList, err := FindAllNodes()
+    if err != nil {
+        return ret, err
+    }
+
+    for _, node := range nodeList {
         for _, dev := range node.NetDevs {
             if dev.Hwaddr == hwa {
-                return node
+                return node, nil
             }
         }
     }
 
-    return ret
+    return ret, errors.New("No nodes found with HW Addr: " + hwa)
 }
 
 
-func FindAllVnfs() []string {
+func FindAllVnfs() ([]string, error) {
     var ret []string
     set := make(map[string]bool)
 
-    for _, node := range FindAllNodes() {
+    nodeList, err := FindAllNodes()
+    if err != nil {
+        return ret, err
+    }
+
+    for _, node := range nodeList {
         if node.Vnfs != "" {
             set[node.Vnfs] = true
         }
@@ -143,16 +151,21 @@ func FindAllVnfs() []string {
         ret = append(ret, entry)
     }
 
-    return ret
+    return ret, nil
 }
 
 
 
-func FindAllKernels() []string {
+func FindAllKernels() ([]string, error) {
     var ret []string
     set := make(map[string]bool)
 
-    for _, node := range FindAllNodes() {
+    nodeList, err := FindAllNodes()
+    if err != nil {
+        return ret, err
+    }
+
+    for _, node := range nodeList {
         if node.KernelVersion != "" {
             set[node.KernelVersion] = true
         }
@@ -162,15 +175,20 @@ func FindAllKernels() []string {
         ret = append(ret, entry)
     }
 
-    return ret
+    return ret, nil
 }
 
 //FindAllOverlays
-func FindAllOverlays() []string {
+func FindAllOverlays() ([]string, error) {
     var ret []string
     set := make(map[string]bool)
 
-    for _, node := range FindAllNodes() {
+    nodeList, err := FindAllNodes()
+    if err != nil {
+        return ret, err
+    }
+
+    for _, node := range nodeList {
         if node.Overlay != "" {
             set[node.Overlay] = true
         }
@@ -180,7 +198,7 @@ func FindAllOverlays() []string {
         ret = append(ret, entry)
     }
 
-    return ret
+    return ret, nil
 }
 
 

internal/pkg/errors/errors.go

@@ -0,0 +1,20 @@
+package errors
+
+
+type error interface {
+	Error() string
+}
+
+// errorString is a trivial implementation of error.
+type errorString struct {
+	s string
+}
+
+func (e *errorString) Error() string {
+	return e.s
+}
+
+// New returns an error that formats as the given text.
+func New(text string) error {
+	return &errorString{text}
+}

overlays/generic/init

@@ -31,7 +31,10 @@ mount -t devtmpfs devtmpfs /dev
 
 /sbin/ip link set dev lo up
 
+nohup /warewulf/bin/wwclient >/var/log/wwclient.log 2>&1 </dev/null &
+
 echo "Calling /sbin/init..."
 echo
 
+sleep 2
 exec /sbin/init