Rework batch package

Use a buffered channel to limit the number of active goroutines without needing
to wait for all active goroutines to finish
This commit is contained in:
Ian Kaneshiro
2020-12-18 19:14:19 -08:00
parent 6f8ec8797b
commit c3281ec333
7 changed files with 47 additions and 59 deletions

View File

@@ -1,37 +1,35 @@
package main
import (
"github.com/hpcng/warewulf/internal/pkg/batch"
"fmt"
"time"
"runtime"
"time"
"github.com/hpcng/warewulf/internal/pkg/batch"
)
func exampleJob(name string, id int) string {
fmt.Printf("[%d] %s started...\n", id, name)
time.Sleep(time.Second * 5)
time.Sleep(time.Second * time.Duration(id))
fmt.Printf("[%d] %s ending...\n", id, name)
return name
}
func main() {
fmt.Printf("GOMAXPROCS=%d\n", runtime.GOMAXPROCS(0))
jobstorun := 1000
jobstorun := 100
// create batch pool to run 100 jobs at a time every 10 seconds
batchpool := batch.New(100, 10)
batchpool := batch.New(10)
retvalues := make(chan string, jobstorun)
// submit a bunch of jobs
for i:=0; i<jobstorun; i++ {
for i := 0; i < jobstorun; i++ {
name := fmt.Sprintf("job-%04d", i)
id := i;
id := i
batchpool.Submit(func() {
retvalues <- exampleJob(name, id)
})
@@ -43,9 +41,8 @@ func main() {
close(retvalues)
for s:= range retvalues {
for s := range retvalues {
fmt.Printf("Return value is %s\n", s)
}
}