From: Sergey Matveev Date: Sat, 21 Nov 2020 14:21:53 +0000 (+0300) Subject: Various refactoring X-Git-Tag: v0.2.0~13 X-Git-Url: http://www.git.cypherpunks.ru/?p=goredo.git;a=commitdiff_plain;h=634ee31ae35a2eaac517e0ffa1b7caede4b27ac5 Various refactoring --- diff --git a/dep.go b/dep.go index 8749f09..6900357 100644 --- a/dep.go +++ b/dep.go @@ -33,6 +33,10 @@ import ( "golang.org/x/crypto/blake2b" ) +const EnvNoHash = "REDO_NO_HASH" + +var NoHash bool + func recfileWrite(fdDep *os.File, fields ...recfile.Field) error { w := recfile.NewWriter(fdDep) if _, err := w.RecordStart(); err != nil { @@ -127,7 +131,7 @@ func writeDeps(fdDep *os.File, tgts []string) error { return nil } ups := []string{} - upLevels := strings.Count(os.Getenv(RedoDirPrefixEnv), "/") + upLevels := strings.Count(os.Getenv(EnvDirPrefix), "/") for i := 0; i < upLevels; i++ { ups = append(ups, "..") } diff --git a/do.go b/do.go index a2f95bf..e120680 100644 --- a/do.go +++ b/do.go @@ -26,7 +26,10 @@ import ( "strings" ) -const TopFile = "top" +const ( + EnvTopDir = "REDO_TOP_DIR" + TopFile = "top" +) var TopDir string diff --git a/ifchange.go b/ifchange.go index 972da4d..73dcae2 100644 --- a/ifchange.go +++ b/ifchange.go @@ -17,6 +17,13 @@ along with this program. If not, see . package main +import "sync" + +var ( + Force bool = false + Jobs sync.WaitGroup +) + func isOkRun(err error) bool { if err == nil { return true diff --git a/js.go b/js.go index d3ed7bb..ccc9fe1 100644 --- a/js.go +++ b/js.go @@ -19,53 +19,103 @@ along with this program. If not, see . package main -import "os" +import ( + "flag" + "log" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" +) + +const ( + EnvJobs = "REDO_JOBS" + EnvJSFd = "REDO_JS_FD" +) var ( - JSR *os.File - JSW *os.File + JSR *os.File + JSW *os.File + jsTokens int + jsTokensM sync.Mutex + + flagJobs = flag.Uint64("j", 1, "number of parallel jobs (0=inf) (REDO_JOBS)") ) func jsInit() { - jsrRaw := os.Getenv(RedoJSRFdEnv) - jswRaw := os.Getenv(RedoJSWFdEnv) - if (jsrRaw == "" && jswRaw != "") || (jsrRaw != "" && jswRaw == "") { - panic("both JSR and JSW must be set") - } - if jsrRaw == "NO" { + jsRaw := os.Getenv(EnvJSFd) + if jsRaw == "NO" { // infinite jobs return } - if jsrRaw != "" { - JSR = mustParseFd(jsrRaw, "JSR") - JSW = mustParseFd(jswRaw, "JSW") + if jsRaw != "" { + cols := strings.Split(jsRaw, ",") + if len(cols) != 2 { + log.Fatalln("invalid", EnvJSFd, "format") + } + JSR = mustParseFd(cols[0], "JSR") + JSW = mustParseFd(cols[1], "JSW") jsRelease("ifchange entered") + + killed := make(chan os.Signal, 0) + signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-killed + jsTokensM.Lock() + for ; jsTokens > 0; jsTokens-- { + jsReleaseNoLock() + } + os.Exit(1) + }() return } - if *JobsN == 0 { + + var jobs uint64 + if v := os.Getenv(EnvJobs); v != "" { + var err error + jobs, err = strconv.ParseUint(v, 10, 64) + if err != nil { + log.Fatalln("can not parse", EnvJobs, err) + } + } else { + jobs = *flagJobs + } + if jobs == 0 { // infinite jobs return } + var err error JSR, JSW, err = os.Pipe() if err != nil { panic(err) } - for i := uint(0); i < *JobsN; i++ { + for i := uint64(0); i < jobs; i++ { jsRelease("initial fill") } } -func jsRelease(ctx string) { - if JSW == nil { - return - } - trace(CJS, "release from %s", ctx) +func jsReleaseNoLock() { if n, err := JSW.Write([]byte{0}); err != nil || n != 1 { panic("can not write JSW") } } +func jsRelease(ctx string) int { + if JSW == nil { + return 0 + } + trace(CJS, "release from %s", ctx) + jsTokensM.Lock() + jsTokens-- + left := jsTokens + jsReleaseNoLock() + jsTokensM.Unlock() + return left +} + func jsAcquire(ctx string) { if JSR == nil { return @@ -74,5 +124,8 @@ func jsAcquire(ctx string) { if n, err := JSR.Read([]byte{0}); err != nil || n != 1 { panic("can not read JSR") } + jsTokensM.Lock() + jsTokens++ + jsTokensM.Unlock() trace(CJS, "acquired for %s", ctx) } diff --git a/log.go b/log.go index 335ca53..f4fa2ca 100644 --- a/log.go +++ b/log.go @@ -20,12 +20,21 @@ along with this program. If not, see . package main import ( + "flag" "fmt" "os" "strings" ) const ( + EnvLevel = "REDO_LEVEL" + EnvDebug = "REDO_DEBUG" + EnvLogWait = "REDO_LOG_WAIT" + EnvLogLock = "REDO_LOG_LOCK" + EnvLogPid = "REDO_LOG_PID" + EnvLogJS = "REDO_LOG_JS" + EnvNoColor = "NO_COLOR" + CReset = "\033[0m" CBold = "\033[1m" CBlack = "\033[30;1m" @@ -48,12 +57,19 @@ const ( ) var ( + Level = 0 NoColor bool Debug bool LogWait bool LogLock bool LogJS bool MyPid int + + flagDebug = flag.Bool("debug", false, "enable debug logging (REDO_DEBUG=1)") + flagLogWait = flag.Bool("log-wait", false, "enable wait messages logging (REDO_LOG_WAIT=1)") + flagLogLock = flag.Bool("log-lock", false, "enable lock messages logging (REDO_LOG_LOCK=1)") + flagLogPid = flag.Bool("log-pid", false, "append PIDs (REDO_LOG_PID=1)") + flagLogJS = flag.Bool("log-js", false, "enable jobserver messages logging (REDO_LOG_JS=1)") ) func trace(level, format string, args ...interface{}) { diff --git a/main.go b/main.go index d0d3ab2..42d83be 100644 --- a/main.go +++ b/main.go @@ -28,39 +28,14 @@ import ( "path" "path/filepath" "strconv" - "sync" "go.cypherpunks.ru/recfile" "golang.org/x/sys/unix" ) -const ( - RedoLevelEnv = "REDO_LEVEL" - RedoDepFdEnv = "REDO_DEP_FD" - RedoDirPrefixEnv = "REDO_DIRPREFIX" - RedoBuildUUIDEnv = "REDO_BUILD_UUID" - RedoStderrPrefixEnv = "REDO_STDERR_PREFIX" - RedoJSRFdEnv = "REDO_JSR_FD" - RedoJSWFdEnv = "REDO_JSW_FD" -) - var ( - Level = 0 - Trace bool = false - Force bool = false - NoHash bool = false - NoSync bool = false - Cwd string - Jobs sync.WaitGroup - - JobsN = flag.Uint("j", 1, "number of parallel jobs (0 for infinite)") - - StderrKeep bool = false - StderrSilent bool = false - StderrPrefix string - + Cwd string BuildUUID string - IsMaster bool = false ) func mustSetenv(key, value string) { @@ -82,14 +57,6 @@ func mustParseFd(v, name string) *os.File { } func main() { - xflag := flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)") - stderrKeep := flag.Bool("stderr-keep", false, "keep job's stderr (REDO_STDERR_KEEP=1)") - stderrSilent := flag.Bool("stderr-silent", false, "do not print job's stderr (REDO_STDERR_SILENT=1)") - debug := flag.Bool("debug", false, "enable debug logging (REDO_DEBUG=1)") - logWait := flag.Bool("log-wait", false, "enable wait messages logging (REDO_LOG_WAIT=1)") - logLock := flag.Bool("log-lock", false, "enable lock messages logging (REDO_LOG_LOCK=1)") - logPid := flag.Bool("log-pid", false, "append PIDs (REDO_LOG_PID=1)") - logJS := flag.Bool("log-js", false, "enable jobserver messages logging (REDO_LOG_JS=1)") version := flag.Bool("version", false, "print version") warranty := flag.Bool("warranty", false, "print warranty information") symlinks := flag.Bool("symlinks", false, "create necessary symlinks in current direcotyr") @@ -105,6 +72,7 @@ func main() { return } if *symlinks { + rc := 0 for _, cmdName := range []string{ "redo", "redo-always", @@ -116,10 +84,11 @@ func main() { "redo-whichdo", } { if err := os.Symlink(os.Args[0], cmdName); err != nil { - log.Fatalln(err) + rc = 1 + log.Println(err) } } - return + os.Exit(rc) } log.SetFlags(0) @@ -129,11 +98,11 @@ func main() { panic(err) } - NoColor = os.Getenv("NO_COLOR") != "" - NoHash = os.Getenv("REDO_NO_HASH") == "1" - NoSync = os.Getenv("REDO_NO_SYNC") == "1" + NoColor = os.Getenv(EnvNoColor) != "" + NoHash = os.Getenv(EnvNoHash) == "1" + NoSync = os.Getenv(EnvNoSync) == "1" - TopDir = os.Getenv("REDO_TOP_DIR") + TopDir = os.Getenv(EnvTopDir) if TopDir != "" { TopDir, err = filepath.Abs(TopDir) if err != nil { @@ -141,58 +110,59 @@ func main() { } } - if *stderrKeep { - mustSetenv("REDO_STDERR_KEEP", "1") + if *flagStderrKeep { + mustSetenv(EnvStderrKeep, "1") } - if *stderrSilent { - mustSetenv("REDO_STDERR_SILENT", "1") + if *flagStderrSilent { + mustSetenv(EnvStderrSilent, "1") } - if *debug { - mustSetenv("REDO_DEBUG", "1") + if *flagDebug { + mustSetenv(EnvDebug, "1") } - if *logWait { - mustSetenv("REDO_LOG_WAIT", "1") + if *flagLogWait { + mustSetenv(EnvLogWait, "1") } - if *logLock { - mustSetenv("REDO_LOG_LOCK", "1") + if *flagLogLock { + mustSetenv(EnvLogLock, "1") } - if *logPid { - mustSetenv("REDO_LOG_PID", "1") + if *flagLogPid { + mustSetenv(EnvLogPid, "1") } - if *logJS { - mustSetenv("REDO_LOG_JS", "1") + if *flagLogJS { + mustSetenv(EnvLogJS, "1") } - StderrKeep = os.Getenv("REDO_STDERR_KEEP") == "1" - StderrSilent = os.Getenv("REDO_STDERR_SILENT") == "1" - Debug = os.Getenv("REDO_DEBUG") == "1" - LogWait = os.Getenv("REDO_LOG_WAIT") == "1" - LogLock = os.Getenv("REDO_LOG_LOCK") == "1" - LogJS = os.Getenv("REDO_LOG_JS") == "1" - if Debug || os.Getenv("REDO_LOG_PID") == "1" { + StderrKeep = os.Getenv(EnvStderrKeep) == "1" + StderrSilent = os.Getenv(EnvStderrSilent) == "1" + Debug = os.Getenv(EnvDebug) == "1" + LogWait = os.Getenv(EnvLogWait) == "1" + LogLock = os.Getenv(EnvLogLock) == "1" + LogJS = os.Getenv(EnvLogJS) == "1" + if Debug || os.Getenv(EnvLogPid) == "1" { MyPid = os.Getpid() } - if *xflag { + if *flagTrace { Trace = true } else { - Trace = os.Getenv("REDO_TRACE") == "1" + Trace = os.Getenv(EnvTrace) == "1" } // Those are internal envs - StderrPrefix = os.Getenv(RedoStderrPrefixEnv) - if v := os.Getenv(RedoLevelEnv); v != "" { - level, err := strconv.ParseUint(v, 10, 64) + StderrPrefix = os.Getenv(EnvStderrPrefix) + if v := os.Getenv(EnvLevel); v != "" { + Level, err = strconv.Atoi(v) if err != nil { panic(err) } - Level = int(level) + if Level < 0 { + panic("negative " + EnvLevel) + } } var fdDep *os.File - if v := os.Getenv(RedoDepFdEnv); v != "" { - fdDep = mustParseFd(v, RedoDepFdEnv) + if v := os.Getenv(EnvDepFd); v != "" { + fdDep = mustParseFd(v, EnvDepFd) } - BuildUUID = os.Getenv(RedoBuildUUIDEnv) + BuildUUID = os.Getenv(EnvBuildUUID) if BuildUUID == "" { - IsMaster = true raw := new([16]byte) if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil { panic(err) @@ -223,7 +193,7 @@ CmdSwitch: writeDeps(fdDep, tgts) case "redo-ifcreate": if fdDep == nil { - log.Fatalln("no", RedoDepFdEnv) + log.Fatalln("no", EnvDepFd) } for _, tgt := range tgts { err = ifcreate(fdDep, tgt) @@ -233,7 +203,7 @@ CmdSwitch: } case "redo-always": if fdDep == nil { - log.Fatalln("no", RedoDepFdEnv) + log.Fatalln("no", EnvDepFd) } err = always(fdDep) case "redo-cleanup": @@ -245,7 +215,7 @@ CmdSwitch: } case "redo-stamp": if fdDep == nil { - log.Fatalln("no", RedoDepFdEnv) + log.Fatalln("no", EnvDepFd) } err = stamp(fdDep, os.Stdin) case "redo-log": diff --git a/run.go b/run.go index 58b9f6d..fab6737 100644 --- a/run.go +++ b/run.go @@ -23,6 +23,7 @@ import ( "bufio" "encoding/hex" "errors" + "flag" "fmt" "io" "os" @@ -38,6 +39,15 @@ import ( ) const ( + EnvDepFd = "REDO_DEP_FD" + EnvDirPrefix = "REDO_DIRPREFIX" + EnvBuildUUID = "REDO_BUILD_UUID" + EnvStderrPrefix = "REDO_STDERR_PREFIX" + EnvTrace = "REDO_TRACE" + EnvStderrKeep = "REDO_LOGS" + EnvStderrSilent = "REDO_SILENT" + EnvNoSync = "REDO_NO_SYNC" + RedoDir = ".redo" LockSuffix = ".lock" DepSuffix = ".dep" @@ -45,6 +55,45 @@ const ( LogSuffix = ".log" ) +var ( + Trace bool = false + NoSync bool = false + StderrKeep bool = false + StderrSilent bool = false + StderrPrefix string + + flagTrace = flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)") + flagStderrKeep = flag.Bool("logs", false, "keep job's stderr (REDO_LOGS=1)") + flagStderrSilent = flag.Bool("silent", false, "do not print job's stderr (REDO_SILENT=1)") +) + +type RunErr struct { + Tgt string + DoFile string + Started *time.Time + Finished *time.Time + Err error +} + +func (e RunErr) Unwrap() error { return e.Err } + +func (e *RunErr) Name() string { + var name string + if e.DoFile == "" { + name = e.Tgt + } else { + name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile) + } + if e.Finished == nil { + return name + } + return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds()) +} + +func (e RunErr) Error() string { + return fmt.Sprintf("%s: %s", e.Name(), e.Err) +} + func mkdirs(pth string) error { if _, err := os.Stat(pth); err == nil { return nil @@ -103,33 +152,6 @@ func isModified(cwd, redoDir, tgt string) (bool, error) { return false, nil } -type RunErr struct { - Tgt string - DoFile string - Started *time.Time - Finished *time.Time - Err error -} - -func (e RunErr) Unwrap() error { return e.Err } - -func (e *RunErr) Name() string { - var name string - if e.DoFile == "" { - name = e.Tgt - } else { - name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile) - } - if e.Finished == nil { - return name - } - return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds()) -} - -func (e RunErr) Error() string { - return fmt.Sprintf("%s: %s", e.Name(), e.Err) -} - func syncDir(dir string) error { fd, err := os.Open(dir) if err != nil { @@ -319,29 +341,26 @@ func runScript(tgt string, errs chan error) error { cmd := exec.Command(cmdName, args...) cmd.Dir = cwd cmd.Stdout = fdStdout - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", RedoLevelEnv, Level+1)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoDirPrefixEnv, dirPrefix)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoBuildUUIDEnv, BuildUUID)) + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID)) childStderrPrefix := tempsuffix() cmd.Env = append(cmd.Env, fmt.Sprintf( - "%s=%s", RedoStderrPrefixEnv, childStderrPrefix, + "%s=%s", EnvStderrPrefix, childStderrPrefix, )) cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) fdNum := 0 - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoDepFdEnv, 3+fdNum)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum)) fdNum++ if JSR == nil { // infinite jobs - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSRFdEnv)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSWFdEnv)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJSFd)) } else { cmd.ExtraFiles = append(cmd.ExtraFiles, JSR) cmd.ExtraFiles = append(cmd.ExtraFiles, JSW) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSRFdEnv, 3+fdNum)) - fdNum++ - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSWFdEnv, 3+fdNum)) - fdNum++ + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1)) + fdNum += 2 } // Preparing stderr diff --git a/usage.go b/usage.go index d5dae5e..d01d85d 100644 --- a/usage.go +++ b/usage.go @@ -71,8 +71,8 @@ You can create them by running: goredo -symlinks. record ifcreate dependency for current target. Unusable outside .do * redo-log target [ | tai64nlocal ] display kept target's stderr with TAI64N timestamped lines. Only the - last build is kept. You must enable stderr keeping with either - -stderr-keep, or REDO_STDERR_KEEP=1 + last build is kept. You must enable stderr keeping with either -logs, + or REDO_LOGS=1 * redo-stamp < [$3] record stamp dependency for current target. Unusable outside .do * redo-whichdo target