--- /dev/null
+@node Jobserver
+@unnumbered Jobserver
+
+Parallel builds are made by utilizing the jobserver protocol. Each job
+have to take so called token and return it when it finishes. Jobserver
+creates the pipe, consisting of read and write files, that are passed
+to each @command{goredo} instance. Job takes the token by reading the
+single byte from that pipe, writing it back for returning. Pipe is
+pre-filled with required number of tokens.
+
+@command{goredo} can be integrated with
+@url{http://www.crufty.net/help/sjg/bmake.htm, bmake} and
+@url{https://www.gnu.org/software/make/, GNU Make} (@command{gmake})
+jobserver protocol. All three of them use the same principle of
+jobserver, but different ways of passing pipe's file descriptors
+numbers to child process.
+
+@env{$REDO_MAKE} environment variable controls the compatibility behaviour:
+
+@table @command
+@item bmake
+Pass @code{-j 1 -J X,Y} arguments through @env{MAKEFLAGS} variable.
+@item gmake
+Pass @code{--jobserver-auth=X,Y} arguments through @env{MAKEFLAGS} variable.
+@item none
+Pass @code{X,Y} arguments through @env{REDO_JS_FD} variable.
+Used by default, if @env{$REDO_MAKE} is not set.
+@end table
"log"
"os"
"os/signal"
+ "regexp"
"strconv"
- "strings"
"sync"
"syscall"
)
const (
- EnvJobs = "REDO_JOBS"
- EnvJSFd = "REDO_JS_FD"
+ EnvMakeFlags = "MAKEFLAGS"
+
+ EnvJSFd = "REDO_JS_FD"
+ EnvJobs = "REDO_JOBS"
+ EnvJSToken = "REDO_JS_TOKEN"
+ EnvMake = "REDO_MAKE"
+
+ MakeTypeNone = "none"
+ MakeTypeBmake = "bmake"
+ MakeTypeGmake = "gmake"
)
var (
- JSR *os.File
- JSW *os.File
- jsTokens int
+ // bmake (NetBSD make)
+ BMakeGoodToken = byte('+')
+ BMakeJSArg = "-j 1 -J "
+ BMakeJSArgRe = regexp.MustCompile(`(.*)\s*-J (\d+),(\d+)\s*(.*)`)
+
+ // GNU Make
+ GMakeJSArg = "--jobserver-auth="
+ GMakeJSArgRe = regexp.MustCompile(`(.*)\s*--jobserver-auth=(\d+),(\d+)\s*(.*)`)
+
+ // dummy make
+ DMakeJSArg = ""
+ DMakeJSArgRe = regexp.MustCompile(`(.*)\s*(\d+),(\d+)\s*(.*)`)
+
+ MakeFlagsName = EnvMakeFlags
+ MakeFlags string
+ MakeJSArg string
+
+ JSR *os.File
+ JSW *os.File
+
+ jsToken byte // got via EnvJSToken
+ jsTokens map[byte]int
jsTokensM sync.Mutex
flagJobs = flag.Int("j", -1, fmt.Sprintf("number of parallel jobs (0=inf, <0=1) (%s)", EnvJobs))
)
-func jsInit() {
- jsRaw := os.Getenv(EnvJSFd)
- if jsRaw == "NO" {
- // infinite jobs
- return
- }
- if jsRaw != "" {
- cols := strings.Split(jsRaw, ",")
- if len(cols) != 2 {
- log.Fatalln("invalid", EnvJSFd, "format")
- }
- JSR = mustParseFd(cols[0], "JSR")
- JSW = mustParseFd(cols[1], "JSW")
- jsRelease("ifchange entered")
-
- killed := make(chan os.Signal, 0)
- signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT)
- go func() {
- <-killed
- jsTokensM.Lock()
- for ; jsTokens > 0; jsTokens-- {
- jsReleaseNoLock()
- }
- os.Exit(1)
- }()
- return
- }
-
+func jsStart(jobsEnv string) {
jobs := uint64(1)
var err error
if *flagJobs == 0 {
jobs = 0
} else if *flagJobs > 0 {
jobs = uint64(*flagJobs)
- } else if v := os.Getenv(EnvJobs); v != "" {
- jobs, err = strconv.ParseUint(v, 10, 64)
+ } else if jobsEnv != "" {
+ jobs, err = strconv.ParseUint(jobsEnv, 10, 64)
if err != nil {
log.Fatalln("can not parse", EnvJobs, err)
}
// infinite jobs
return
}
-
JSR, JSW, err = os.Pipe()
if err != nil {
log.Fatalln(err)
}
- for i := uint64(0); i < jobs; i++ {
- jsRelease("initial fill")
+ trace(CJS, "initial fill with %d", jobs)
+ jsTokens[BMakeGoodToken] = int(jobs)
+ for ; jobs > 0; jobs-- {
+ jsReleaseNoLock(BMakeGoodToken)
+ }
+}
+
+func jsInit() {
+ jsTokens = make(map[byte]int)
+
+ makeType := os.Getenv(EnvMake)
+ var makeArgRe *regexp.Regexp
+ switch makeType {
+ case MakeTypeGmake:
+ makeArgRe = GMakeJSArgRe
+ MakeJSArg = GMakeJSArg
+ case MakeTypeBmake:
+ makeArgRe = BMakeJSArgRe
+ MakeJSArg = BMakeJSArg
+ case "":
+ fallthrough
+ case MakeTypeNone:
+ MakeFlagsName = EnvJSFd
+ makeArgRe = DMakeJSArgRe
+ MakeJSArg = DMakeJSArg
+ default:
+ log.Fatalln("unknown", EnvMake, "type")
+ }
+
+ MakeFlags = os.Getenv(MakeFlagsName)
+ jobsEnv := os.Getenv(EnvJobs)
+ if jobsEnv == "NO" {
+ // jobserver disabled, infinite jobs
+ return
+ }
+ if MakeFlags == "" {
+ // we are not running under make
+ jsStart(jobsEnv)
+ return
}
+
+ match := makeArgRe.FindStringSubmatch(MakeFlags)
+ if len(match) == 0 {
+ // MAKEFLAGS does not contain anything related to jobserver
+ jsStart(jobsEnv)
+ return
+ }
+ MakeFlags = match[1] + " " + match[4]
+
+ func() {
+ defer func() {
+ if err := recover(); err != nil {
+ log.Fatalln(err)
+ }
+ }()
+ JSR = mustParseFd(match[2], "JSR")
+ JSW = mustParseFd(match[3], "JSW")
+ }()
+
+ if token := os.Getenv(EnvJSToken); token != "" {
+ jsTokenInt, err := strconv.ParseUint(token, 10, 8)
+ if err != nil {
+ log.Fatalln("invalid", EnvJSToken, "format:", err)
+ }
+ jsToken = byte(jsTokenInt)
+ jsTokens[jsToken]++
+ jsRelease("ifchange entered", jsToken)
+ }
+
+ killed := make(chan os.Signal, 0)
+ signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT)
+ go func() {
+ <-killed
+ jsTokensM.Lock()
+ for token, i := range jsTokens {
+ for ; i > 0; i-- {
+ jsReleaseNoLock(token)
+ }
+ }
+ jsTokensM.Unlock()
+ os.Exit(1)
+ }()
}
-func jsReleaseNoLock() {
- if n, err := JSW.Write([]byte{0}); err != nil || n != 1 {
+func jsReleaseNoLock(token byte) {
+ if n, err := JSW.Write([]byte{token}); err != nil || n != 1 {
log.Fatalln("can not write JSW:", err)
}
}
-func jsRelease(ctx string) int {
+func jsRelease(ctx string, token byte) {
if JSW == nil {
- return 0
+ return
}
trace(CJS, "release from %s", ctx)
jsTokensM.Lock()
- jsTokens--
- left := jsTokens
- jsReleaseNoLock()
+ jsTokens[token]--
+ jsReleaseNoLock(token)
jsTokensM.Unlock()
- return left
}
-func jsAcquire(ctx string) {
+func jsAcquire(ctx string) byte {
if JSR == nil {
- return
+ return BMakeGoodToken
}
trace(CJS, "acquire for %s", ctx)
- if n, err := JSR.Read([]byte{0}); err != nil || n != 1 {
+ token := []byte{0}
+ if n, err := JSR.Read(token); err != nil || n != 1 {
log.Fatalln("can not read JSR:", err)
}
jsTokensM.Lock()
- jsTokens++
+ jsTokens[token[0]]++
jsTokensM.Unlock()
trace(CJS, "acquired for %s", ctx)
+ return token[0]
}
fdNum++
}
- if JSR == nil {
- // infinite jobs
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJSFd))
- } else {
- cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
- cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
- cmd.Env = append(cmd.Env, fmt.Sprintf(
- "%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1,
- ))
- fdNum += 2
- }
-
// Preparing stderr
var fdStderr *os.File
if StderrKeep {
Jobs.Add(1)
go func() {
- jsAcquire(shCtx)
+ jsToken := jsAcquire(shCtx)
+ if JSR == nil {
+ // infinite jobs
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJobs))
+ } else {
+ cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
+ cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
+ cmd.Env = append(cmd.Env, fmt.Sprintf(
+ "%s=%s %s%d,%d",
+ MakeFlagsName, MakeFlags, MakeJSArg, 3+fdNum+0, 3+fdNum+1,
+ ))
+ fdNum += 2
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvJSToken, jsToken))
+ }
+
if FdStatus != nil {
FdStatus.Write([]byte{StatusRun})
}
fdNum++
defer func() {
- jsRelease(shCtx)
+ jsRelease(shCtx, jsToken)
lockRelease()
fdDep.Close()
fdStdout.Close()