From c41fe0fbd726f397636b569161f286821e221885 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Wed, 23 Jun 2021 14:43:11 +0300 Subject: [PATCH] bmake/gmake jobserver protocol compatibility --- doc/features.texi | 1 + doc/index.texi | 1 + doc/jobserver.texi | 28 +++++++ doc/news.texi | 9 +++ js.go | 180 ++++++++++++++++++++++++++++++++------------- run.go | 30 ++++---- usage.go | 5 +- 7 files changed, 187 insertions(+), 67 deletions(-) create mode 100644 doc/jobserver.texi diff --git a/doc/features.texi b/doc/features.texi index df1778b..48c9adc 100644 --- a/doc/features.texi +++ b/doc/features.texi @@ -33,5 +33,6 @@ implementations. @command{tai64nlocal} utility from @url{http://cr.yp.to/daemontools.html, daemontools}, or similar one: @code{go get go.cypherpunks.ru/tai64n/cmd/tai64nlocal} +@item Either GNU Make or bmake @ref{Jobserver, jobserver} compatibility support @end itemize diff --git a/doc/index.texi b/doc/index.texi index c7aa0eb..131c301 100644 --- a/doc/index.texi +++ b/doc/index.texi @@ -53,6 +53,7 @@ maillist. Announcements also go to this mailing list. @include install.texi @include faq.texi @include state.texi +@include jobserver.texi @include thanks.texi @bye diff --git a/doc/jobserver.texi b/doc/jobserver.texi new file mode 100644 index 0000000..0bb3ce5 --- /dev/null +++ b/doc/jobserver.texi @@ -0,0 +1,28 @@ +@node Jobserver +@unnumbered Jobserver + +Parallel builds are made by utilizing the jobserver protocol. Each job +have to take so called token and return it when it finishes. Jobserver +creates the pipe, consisting of read and write files, that are passed +to each @command{goredo} instance. Job takes the token by reading the +single byte from that pipe, writing it back for returning. Pipe is +pre-filled with required number of tokens. + +@command{goredo} can be integrated with +@url{http://www.crufty.net/help/sjg/bmake.htm, bmake} and +@url{https://www.gnu.org/software/make/, GNU Make} (@command{gmake}) +jobserver protocol. All three of them use the same principle of +jobserver, but different ways of passing pipe's file descriptors +numbers to child process. + +@env{$REDO_MAKE} environment variable controls the compatibility behaviour: + +@table @command +@item bmake +Pass @code{-j 1 -J X,Y} arguments through @env{MAKEFLAGS} variable. +@item gmake +Pass @code{--jobserver-auth=X,Y} arguments through @env{MAKEFLAGS} variable. +@item none +Pass @code{X,Y} arguments through @env{REDO_JS_FD} variable. +Used by default, if @env{$REDO_MAKE} is not set. +@end table diff --git a/doc/news.texi b/doc/news.texi index d301bdd..a0ba076 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -1,6 +1,15 @@ @node News @unnumbered News +@anchor{Release 1.7.0} +@section Release 1.7.0 +@itemize +@item + Optional compatibility (through @env{REDO_MAKE=@{bmake|gmake@}}) + with either NetBSD's bmake or GNU Make jobserver protocols, being + able to tightly integrate @command{goredo} with the @command{make}. +@end itemize + @anchor{Release 1.6.0} @section Release 1.6.0 @itemize diff --git a/js.go b/js.go index 701d26d..1e452ee 100644 --- a/js.go +++ b/js.go @@ -25,62 +25,62 @@ import ( "log" "os" "os/signal" + "regexp" "strconv" - "strings" "sync" "syscall" ) const ( - EnvJobs = "REDO_JOBS" - EnvJSFd = "REDO_JS_FD" + EnvMakeFlags = "MAKEFLAGS" + + EnvJSFd = "REDO_JS_FD" + EnvJobs = "REDO_JOBS" + EnvJSToken = "REDO_JS_TOKEN" + EnvMake = "REDO_MAKE" + + MakeTypeNone = "none" + MakeTypeBmake = "bmake" + MakeTypeGmake = "gmake" ) var ( - JSR *os.File - JSW *os.File - jsTokens int + // bmake (NetBSD make) + BMakeGoodToken = byte('+') + BMakeJSArg = "-j 1 -J " + BMakeJSArgRe = regexp.MustCompile(`(.*)\s*-J (\d+),(\d+)\s*(.*)`) + + // GNU Make + GMakeJSArg = "--jobserver-auth=" + GMakeJSArgRe = regexp.MustCompile(`(.*)\s*--jobserver-auth=(\d+),(\d+)\s*(.*)`) + + // dummy make + DMakeJSArg = "" + DMakeJSArgRe = regexp.MustCompile(`(.*)\s*(\d+),(\d+)\s*(.*)`) + + MakeFlagsName = EnvMakeFlags + MakeFlags string + MakeJSArg string + + JSR *os.File + JSW *os.File + + jsToken byte // got via EnvJSToken + jsTokens map[byte]int jsTokensM sync.Mutex flagJobs = flag.Int("j", -1, fmt.Sprintf("number of parallel jobs (0=inf, <0=1) (%s)", EnvJobs)) ) -func jsInit() { - jsRaw := os.Getenv(EnvJSFd) - if jsRaw == "NO" { - // infinite jobs - return - } - if jsRaw != "" { - cols := strings.Split(jsRaw, ",") - if len(cols) != 2 { - log.Fatalln("invalid", EnvJSFd, "format") - } - JSR = mustParseFd(cols[0], "JSR") - JSW = mustParseFd(cols[1], "JSW") - jsRelease("ifchange entered") - - killed := make(chan os.Signal, 0) - signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT) - go func() { - <-killed - jsTokensM.Lock() - for ; jsTokens > 0; jsTokens-- { - jsReleaseNoLock() - } - os.Exit(1) - }() - return - } - +func jsStart(jobsEnv string) { jobs := uint64(1) var err error if *flagJobs == 0 { jobs = 0 } else if *flagJobs > 0 { jobs = uint64(*flagJobs) - } else if v := os.Getenv(EnvJobs); v != "" { - jobs, err = strconv.ParseUint(v, 10, 64) + } else if jobsEnv != "" { + jobs, err = strconv.ParseUint(jobsEnv, 10, 64) if err != nil { log.Fatalln("can not parse", EnvJobs, err) } @@ -89,45 +89,123 @@ func jsInit() { // infinite jobs return } - JSR, JSW, err = os.Pipe() if err != nil { log.Fatalln(err) } - for i := uint64(0); i < jobs; i++ { - jsRelease("initial fill") + trace(CJS, "initial fill with %d", jobs) + jsTokens[BMakeGoodToken] = int(jobs) + for ; jobs > 0; jobs-- { + jsReleaseNoLock(BMakeGoodToken) + } +} + +func jsInit() { + jsTokens = make(map[byte]int) + + makeType := os.Getenv(EnvMake) + var makeArgRe *regexp.Regexp + switch makeType { + case MakeTypeGmake: + makeArgRe = GMakeJSArgRe + MakeJSArg = GMakeJSArg + case MakeTypeBmake: + makeArgRe = BMakeJSArgRe + MakeJSArg = BMakeJSArg + case "": + fallthrough + case MakeTypeNone: + MakeFlagsName = EnvJSFd + makeArgRe = DMakeJSArgRe + MakeJSArg = DMakeJSArg + default: + log.Fatalln("unknown", EnvMake, "type") + } + + MakeFlags = os.Getenv(MakeFlagsName) + jobsEnv := os.Getenv(EnvJobs) + if jobsEnv == "NO" { + // jobserver disabled, infinite jobs + return + } + if MakeFlags == "" { + // we are not running under make + jsStart(jobsEnv) + return } + + match := makeArgRe.FindStringSubmatch(MakeFlags) + if len(match) == 0 { + // MAKEFLAGS does not contain anything related to jobserver + jsStart(jobsEnv) + return + } + MakeFlags = match[1] + " " + match[4] + + func() { + defer func() { + if err := recover(); err != nil { + log.Fatalln(err) + } + }() + JSR = mustParseFd(match[2], "JSR") + JSW = mustParseFd(match[3], "JSW") + }() + + if token := os.Getenv(EnvJSToken); token != "" { + jsTokenInt, err := strconv.ParseUint(token, 10, 8) + if err != nil { + log.Fatalln("invalid", EnvJSToken, "format:", err) + } + jsToken = byte(jsTokenInt) + jsTokens[jsToken]++ + jsRelease("ifchange entered", jsToken) + } + + killed := make(chan os.Signal, 0) + signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-killed + jsTokensM.Lock() + for token, i := range jsTokens { + for ; i > 0; i-- { + jsReleaseNoLock(token) + } + } + jsTokensM.Unlock() + os.Exit(1) + }() } -func jsReleaseNoLock() { - if n, err := JSW.Write([]byte{0}); err != nil || n != 1 { +func jsReleaseNoLock(token byte) { + if n, err := JSW.Write([]byte{token}); err != nil || n != 1 { log.Fatalln("can not write JSW:", err) } } -func jsRelease(ctx string) int { +func jsRelease(ctx string, token byte) { if JSW == nil { - return 0 + return } trace(CJS, "release from %s", ctx) jsTokensM.Lock() - jsTokens-- - left := jsTokens - jsReleaseNoLock() + jsTokens[token]-- + jsReleaseNoLock(token) jsTokensM.Unlock() - return left } -func jsAcquire(ctx string) { +func jsAcquire(ctx string) byte { if JSR == nil { - return + return BMakeGoodToken } trace(CJS, "acquire for %s", ctx) - if n, err := JSR.Read([]byte{0}); err != nil || n != 1 { + token := []byte{0} + if n, err := JSR.Read(token); err != nil || n != 1 { log.Fatalln("can not read JSR:", err) } jsTokensM.Lock() - jsTokens++ + jsTokens[token[0]]++ jsTokensM.Unlock() trace(CJS, "acquired for %s", ctx) + return token[0] } diff --git a/run.go b/run.go index 4ca90d6..1c1b0d7 100644 --- a/run.go +++ b/run.go @@ -362,18 +362,6 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { fdNum++ } - if JSR == nil { - // infinite jobs - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJSFd)) - } else { - cmd.ExtraFiles = append(cmd.ExtraFiles, JSR) - cmd.ExtraFiles = append(cmd.ExtraFiles, JSW) - cmd.Env = append(cmd.Env, fmt.Sprintf( - "%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1, - )) - fdNum += 2 - } - // Preparing stderr var fdStderr *os.File if StderrKeep { @@ -396,7 +384,21 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { Jobs.Add(1) go func() { - jsAcquire(shCtx) + jsToken := jsAcquire(shCtx) + if JSR == nil { + // infinite jobs + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJobs)) + } else { + cmd.ExtraFiles = append(cmd.ExtraFiles, JSR) + cmd.ExtraFiles = append(cmd.ExtraFiles, JSW) + cmd.Env = append(cmd.Env, fmt.Sprintf( + "%s=%s %s%d,%d", + MakeFlagsName, MakeFlags, MakeJSArg, 3+fdNum+0, 3+fdNum+1, + )) + fdNum += 2 + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvJSToken, jsToken)) + } + if FdStatus != nil { FdStatus.Write([]byte{StatusRun}) } @@ -421,7 +423,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { fdNum++ defer func() { - jsRelease(shCtx) + jsRelease(shCtx, jsToken) lockRelease() fdDep.Close() fdStdout.Close() diff --git a/usage.go b/usage.go index b18c93b..56ccf57 100644 --- a/usage.go +++ b/usage.go @@ -24,7 +24,7 @@ import ( ) const ( - Version = "1.6.0" + Version = "1.7.0" Warranty = `Copyright (C) 2020-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify @@ -120,5 +120,6 @@ Additional environment variables: REDO_TOP_DIR -- do not search for .do above that directory (it can contain .redo/top as an alternative) REDO_INODE_NO_TRUST -- do not trust inode information (except for size) - and always check file's hash`) + and always check file's hash + REDO_MAKE -- bmake/gmake/none(default) jobserver protocol compatibility`) } -- 2.44.0