"golang.org/x/crypto/blake2b"
)
+const EnvNoHash = "REDO_NO_HASH"
+
+var NoHash bool
+
func recfileWrite(fdDep *os.File, fields ...recfile.Field) error {
w := recfile.NewWriter(fdDep)
if _, err := w.RecordStart(); err != nil {
return nil
}
ups := []string{}
- upLevels := strings.Count(os.Getenv(RedoDirPrefixEnv), "/")
+ upLevels := strings.Count(os.Getenv(EnvDirPrefix), "/")
for i := 0; i < upLevels; i++ {
ups = append(ups, "..")
}
"strings"
)
-const TopFile = "top"
+const (
+ EnvTopDir = "REDO_TOP_DIR"
+ TopFile = "top"
+)
var TopDir string
package main
+import "sync"
+
+var (
+ Force bool = false
+ Jobs sync.WaitGroup
+)
+
func isOkRun(err error) bool {
if err == nil {
return true
package main
-import "os"
+import (
+ "flag"
+ "log"
+ "os"
+ "os/signal"
+ "strconv"
+ "strings"
+ "sync"
+ "syscall"
+)
+
+const (
+ EnvJobs = "REDO_JOBS"
+ EnvJSFd = "REDO_JS_FD"
+)
var (
- JSR *os.File
- JSW *os.File
+ JSR *os.File
+ JSW *os.File
+ jsTokens int
+ jsTokensM sync.Mutex
+
+ flagJobs = flag.Uint64("j", 1, "number of parallel jobs (0=inf) (REDO_JOBS)")
)
func jsInit() {
- jsrRaw := os.Getenv(RedoJSRFdEnv)
- jswRaw := os.Getenv(RedoJSWFdEnv)
- if (jsrRaw == "" && jswRaw != "") || (jsrRaw != "" && jswRaw == "") {
- panic("both JSR and JSW must be set")
- }
- if jsrRaw == "NO" {
+ jsRaw := os.Getenv(EnvJSFd)
+ if jsRaw == "NO" {
// infinite jobs
return
}
- if jsrRaw != "" {
- JSR = mustParseFd(jsrRaw, "JSR")
- JSW = mustParseFd(jswRaw, "JSW")
+ if jsRaw != "" {
+ cols := strings.Split(jsRaw, ",")
+ if len(cols) != 2 {
+ log.Fatalln("invalid", EnvJSFd, "format")
+ }
+ JSR = mustParseFd(cols[0], "JSR")
+ JSW = mustParseFd(cols[1], "JSW")
jsRelease("ifchange entered")
+
+ killed := make(chan os.Signal, 0)
+ signal.Notify(killed, syscall.SIGTERM, syscall.SIGINT)
+ go func() {
+ <-killed
+ jsTokensM.Lock()
+ for ; jsTokens > 0; jsTokens-- {
+ jsReleaseNoLock()
+ }
+ os.Exit(1)
+ }()
return
}
- if *JobsN == 0 {
+
+ var jobs uint64
+ if v := os.Getenv(EnvJobs); v != "" {
+ var err error
+ jobs, err = strconv.ParseUint(v, 10, 64)
+ if err != nil {
+ log.Fatalln("can not parse", EnvJobs, err)
+ }
+ } else {
+ jobs = *flagJobs
+ }
+ if jobs == 0 {
// infinite jobs
return
}
+
var err error
JSR, JSW, err = os.Pipe()
if err != nil {
panic(err)
}
- for i := uint(0); i < *JobsN; i++ {
+ for i := uint64(0); i < jobs; i++ {
jsRelease("initial fill")
}
}
-func jsRelease(ctx string) {
- if JSW == nil {
- return
- }
- trace(CJS, "release from %s", ctx)
+func jsReleaseNoLock() {
if n, err := JSW.Write([]byte{0}); err != nil || n != 1 {
panic("can not write JSW")
}
}
+func jsRelease(ctx string) int {
+ if JSW == nil {
+ return 0
+ }
+ trace(CJS, "release from %s", ctx)
+ jsTokensM.Lock()
+ jsTokens--
+ left := jsTokens
+ jsReleaseNoLock()
+ jsTokensM.Unlock()
+ return left
+}
+
func jsAcquire(ctx string) {
if JSR == nil {
return
if n, err := JSR.Read([]byte{0}); err != nil || n != 1 {
panic("can not read JSR")
}
+ jsTokensM.Lock()
+ jsTokens++
+ jsTokensM.Unlock()
trace(CJS, "acquired for %s", ctx)
}
package main
import (
+ "flag"
"fmt"
"os"
"strings"
)
const (
+ EnvLevel = "REDO_LEVEL"
+ EnvDebug = "REDO_DEBUG"
+ EnvLogWait = "REDO_LOG_WAIT"
+ EnvLogLock = "REDO_LOG_LOCK"
+ EnvLogPid = "REDO_LOG_PID"
+ EnvLogJS = "REDO_LOG_JS"
+ EnvNoColor = "NO_COLOR"
+
CReset = "\033[0m"
CBold = "\033[1m"
CBlack = "\033[30;1m"
)
var (
+ Level = 0
NoColor bool
Debug bool
LogWait bool
LogLock bool
LogJS bool
MyPid int
+
+ flagDebug = flag.Bool("debug", false, "enable debug logging (REDO_DEBUG=1)")
+ flagLogWait = flag.Bool("log-wait", false, "enable wait messages logging (REDO_LOG_WAIT=1)")
+ flagLogLock = flag.Bool("log-lock", false, "enable lock messages logging (REDO_LOG_LOCK=1)")
+ flagLogPid = flag.Bool("log-pid", false, "append PIDs (REDO_LOG_PID=1)")
+ flagLogJS = flag.Bool("log-js", false, "enable jobserver messages logging (REDO_LOG_JS=1)")
)
func trace(level, format string, args ...interface{}) {
"path"
"path/filepath"
"strconv"
- "sync"
"go.cypherpunks.ru/recfile"
"golang.org/x/sys/unix"
)
-const (
- RedoLevelEnv = "REDO_LEVEL"
- RedoDepFdEnv = "REDO_DEP_FD"
- RedoDirPrefixEnv = "REDO_DIRPREFIX"
- RedoBuildUUIDEnv = "REDO_BUILD_UUID"
- RedoStderrPrefixEnv = "REDO_STDERR_PREFIX"
- RedoJSRFdEnv = "REDO_JSR_FD"
- RedoJSWFdEnv = "REDO_JSW_FD"
-)
-
var (
- Level = 0
- Trace bool = false
- Force bool = false
- NoHash bool = false
- NoSync bool = false
- Cwd string
- Jobs sync.WaitGroup
-
- JobsN = flag.Uint("j", 1, "number of parallel jobs (0 for infinite)")
-
- StderrKeep bool = false
- StderrSilent bool = false
- StderrPrefix string
-
+ Cwd string
BuildUUID string
- IsMaster bool = false
)
func mustSetenv(key, value string) {
}
func main() {
- xflag := flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)")
- stderrKeep := flag.Bool("stderr-keep", false, "keep job's stderr (REDO_STDERR_KEEP=1)")
- stderrSilent := flag.Bool("stderr-silent", false, "do not print job's stderr (REDO_STDERR_SILENT=1)")
- debug := flag.Bool("debug", false, "enable debug logging (REDO_DEBUG=1)")
- logWait := flag.Bool("log-wait", false, "enable wait messages logging (REDO_LOG_WAIT=1)")
- logLock := flag.Bool("log-lock", false, "enable lock messages logging (REDO_LOG_LOCK=1)")
- logPid := flag.Bool("log-pid", false, "append PIDs (REDO_LOG_PID=1)")
- logJS := flag.Bool("log-js", false, "enable jobserver messages logging (REDO_LOG_JS=1)")
version := flag.Bool("version", false, "print version")
warranty := flag.Bool("warranty", false, "print warranty information")
symlinks := flag.Bool("symlinks", false, "create necessary symlinks in current direcotyr")
return
}
if *symlinks {
+ rc := 0
for _, cmdName := range []string{
"redo",
"redo-always",
"redo-whichdo",
} {
if err := os.Symlink(os.Args[0], cmdName); err != nil {
- log.Fatalln(err)
+ rc = 1
+ log.Println(err)
}
}
- return
+ os.Exit(rc)
}
log.SetFlags(0)
panic(err)
}
- NoColor = os.Getenv("NO_COLOR") != ""
- NoHash = os.Getenv("REDO_NO_HASH") == "1"
- NoSync = os.Getenv("REDO_NO_SYNC") == "1"
+ NoColor = os.Getenv(EnvNoColor) != ""
+ NoHash = os.Getenv(EnvNoHash) == "1"
+ NoSync = os.Getenv(EnvNoSync) == "1"
- TopDir = os.Getenv("REDO_TOP_DIR")
+ TopDir = os.Getenv(EnvTopDir)
if TopDir != "" {
TopDir, err = filepath.Abs(TopDir)
if err != nil {
}
}
- if *stderrKeep {
- mustSetenv("REDO_STDERR_KEEP", "1")
+ if *flagStderrKeep {
+ mustSetenv(EnvStderrKeep, "1")
}
- if *stderrSilent {
- mustSetenv("REDO_STDERR_SILENT", "1")
+ if *flagStderrSilent {
+ mustSetenv(EnvStderrSilent, "1")
}
- if *debug {
- mustSetenv("REDO_DEBUG", "1")
+ if *flagDebug {
+ mustSetenv(EnvDebug, "1")
}
- if *logWait {
- mustSetenv("REDO_LOG_WAIT", "1")
+ if *flagLogWait {
+ mustSetenv(EnvLogWait, "1")
}
- if *logLock {
- mustSetenv("REDO_LOG_LOCK", "1")
+ if *flagLogLock {
+ mustSetenv(EnvLogLock, "1")
}
- if *logPid {
- mustSetenv("REDO_LOG_PID", "1")
+ if *flagLogPid {
+ mustSetenv(EnvLogPid, "1")
}
- if *logJS {
- mustSetenv("REDO_LOG_JS", "1")
+ if *flagLogJS {
+ mustSetenv(EnvLogJS, "1")
}
- StderrKeep = os.Getenv("REDO_STDERR_KEEP") == "1"
- StderrSilent = os.Getenv("REDO_STDERR_SILENT") == "1"
- Debug = os.Getenv("REDO_DEBUG") == "1"
- LogWait = os.Getenv("REDO_LOG_WAIT") == "1"
- LogLock = os.Getenv("REDO_LOG_LOCK") == "1"
- LogJS = os.Getenv("REDO_LOG_JS") == "1"
- if Debug || os.Getenv("REDO_LOG_PID") == "1" {
+ StderrKeep = os.Getenv(EnvStderrKeep) == "1"
+ StderrSilent = os.Getenv(EnvStderrSilent) == "1"
+ Debug = os.Getenv(EnvDebug) == "1"
+ LogWait = os.Getenv(EnvLogWait) == "1"
+ LogLock = os.Getenv(EnvLogLock) == "1"
+ LogJS = os.Getenv(EnvLogJS) == "1"
+ if Debug || os.Getenv(EnvLogPid) == "1" {
MyPid = os.Getpid()
}
- if *xflag {
+ if *flagTrace {
Trace = true
} else {
- Trace = os.Getenv("REDO_TRACE") == "1"
+ Trace = os.Getenv(EnvTrace) == "1"
}
// Those are internal envs
- StderrPrefix = os.Getenv(RedoStderrPrefixEnv)
- if v := os.Getenv(RedoLevelEnv); v != "" {
- level, err := strconv.ParseUint(v, 10, 64)
+ StderrPrefix = os.Getenv(EnvStderrPrefix)
+ if v := os.Getenv(EnvLevel); v != "" {
+ Level, err = strconv.Atoi(v)
if err != nil {
panic(err)
}
- Level = int(level)
+ if Level < 0 {
+ panic("negative " + EnvLevel)
+ }
}
var fdDep *os.File
- if v := os.Getenv(RedoDepFdEnv); v != "" {
- fdDep = mustParseFd(v, RedoDepFdEnv)
+ if v := os.Getenv(EnvDepFd); v != "" {
+ fdDep = mustParseFd(v, EnvDepFd)
}
- BuildUUID = os.Getenv(RedoBuildUUIDEnv)
+ BuildUUID = os.Getenv(EnvBuildUUID)
if BuildUUID == "" {
- IsMaster = true
raw := new([16]byte)
if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil {
panic(err)
writeDeps(fdDep, tgts)
case "redo-ifcreate":
if fdDep == nil {
- log.Fatalln("no", RedoDepFdEnv)
+ log.Fatalln("no", EnvDepFd)
}
for _, tgt := range tgts {
err = ifcreate(fdDep, tgt)
}
case "redo-always":
if fdDep == nil {
- log.Fatalln("no", RedoDepFdEnv)
+ log.Fatalln("no", EnvDepFd)
}
err = always(fdDep)
case "redo-cleanup":
}
case "redo-stamp":
if fdDep == nil {
- log.Fatalln("no", RedoDepFdEnv)
+ log.Fatalln("no", EnvDepFd)
}
err = stamp(fdDep, os.Stdin)
case "redo-log":
"bufio"
"encoding/hex"
"errors"
+ "flag"
"fmt"
"io"
"os"
)
const (
+ EnvDepFd = "REDO_DEP_FD"
+ EnvDirPrefix = "REDO_DIRPREFIX"
+ EnvBuildUUID = "REDO_BUILD_UUID"
+ EnvStderrPrefix = "REDO_STDERR_PREFIX"
+ EnvTrace = "REDO_TRACE"
+ EnvStderrKeep = "REDO_LOGS"
+ EnvStderrSilent = "REDO_SILENT"
+ EnvNoSync = "REDO_NO_SYNC"
+
RedoDir = ".redo"
LockSuffix = ".lock"
DepSuffix = ".dep"
LogSuffix = ".log"
)
+var (
+ Trace bool = false
+ NoSync bool = false
+ StderrKeep bool = false
+ StderrSilent bool = false
+ StderrPrefix string
+
+ flagTrace = flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)")
+ flagStderrKeep = flag.Bool("logs", false, "keep job's stderr (REDO_LOGS=1)")
+ flagStderrSilent = flag.Bool("silent", false, "do not print job's stderr (REDO_SILENT=1)")
+)
+
+type RunErr struct {
+ Tgt string
+ DoFile string
+ Started *time.Time
+ Finished *time.Time
+ Err error
+}
+
+func (e RunErr) Unwrap() error { return e.Err }
+
+func (e *RunErr) Name() string {
+ var name string
+ if e.DoFile == "" {
+ name = e.Tgt
+ } else {
+ name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile)
+ }
+ if e.Finished == nil {
+ return name
+ }
+ return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds())
+}
+
+func (e RunErr) Error() string {
+ return fmt.Sprintf("%s: %s", e.Name(), e.Err)
+}
+
func mkdirs(pth string) error {
if _, err := os.Stat(pth); err == nil {
return nil
return false, nil
}
-type RunErr struct {
- Tgt string
- DoFile string
- Started *time.Time
- Finished *time.Time
- Err error
-}
-
-func (e RunErr) Unwrap() error { return e.Err }
-
-func (e *RunErr) Name() string {
- var name string
- if e.DoFile == "" {
- name = e.Tgt
- } else {
- name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile)
- }
- if e.Finished == nil {
- return name
- }
- return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds())
-}
-
-func (e RunErr) Error() string {
- return fmt.Sprintf("%s: %s", e.Name(), e.Err)
-}
-
func syncDir(dir string) error {
fd, err := os.Open(dir)
if err != nil {
cmd := exec.Command(cmdName, args...)
cmd.Dir = cwd
cmd.Stdout = fdStdout
- cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", RedoLevelEnv, Level+1))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoDirPrefixEnv, dirPrefix))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoBuildUUIDEnv, BuildUUID))
+ cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID))
childStderrPrefix := tempsuffix()
cmd.Env = append(cmd.Env, fmt.Sprintf(
- "%s=%s", RedoStderrPrefixEnv, childStderrPrefix,
+ "%s=%s", EnvStderrPrefix, childStderrPrefix,
))
cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep)
fdNum := 0
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoDepFdEnv, 3+fdNum))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum))
fdNum++
if JSR == nil {
// infinite jobs
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSRFdEnv))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSWFdEnv))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJSFd))
} else {
cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSRFdEnv, 3+fdNum))
- fdNum++
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSWFdEnv, 3+fdNum))
- fdNum++
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1))
+ fdNum += 2
}
// Preparing stderr
record ifcreate dependency for current target. Unusable outside .do
* redo-log target [ | tai64nlocal ]
display kept target's stderr with TAI64N timestamped lines. Only the
- last build is kept. You must enable stderr keeping with either
- -stderr-keep, or REDO_STDERR_KEEP=1
+ last build is kept. You must enable stderr keeping with either -logs,
+ or REDO_LOGS=1
* redo-stamp < [$3]
record stamp dependency for current target. Unusable outside .do
* redo-whichdo target