/*
-goredo -- redo implementation on pure Go
-Copyright (C) 2020 Sergey Matveev <stargrave@stargrave.org>
+goredo -- djb's redo implementation on pure Go
+Copyright (C) 2020-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
import (
"bufio"
+ "crypto/rand"
"encoding/hex"
"errors"
+ "flag"
"fmt"
"io"
+ "io/fs"
+ "log"
"os"
"os/exec"
"path"
"strconv"
"strings"
+ "sync"
"syscall"
"time"
"go.cypherpunks.ru/recfile"
+ "go.cypherpunks.ru/tai64n/v2"
"golang.org/x/sys/unix"
)
const (
- RedoDir = ".redo"
- LockSuffix = ".lock"
- DepSuffix = ".dep"
- TmpPrefix = ".redo."
- LogSuffix = ".log"
+ EnvDepFd = "REDO_DEP_FD"
+ EnvDirPrefix = "REDO_DIRPREFIX"
+ EnvDepCwd = "REDO_DEP_CWD"
+ EnvBuildUUID = "REDO_BUILD_UUID"
+ EnvStderrPrefix = "REDO_STDERR_PREFIX"
+ EnvTrace = "REDO_TRACE"
+ EnvStderrKeep = "REDO_LOGS"
+ EnvStderrSilent = "REDO_SILENT"
+ EnvNoSync = "REDO_NO_SYNC"
+ EnvStopIfMod = "REDO_STOP_IF_MODIFIED"
+
+ RedoDir = ".redo"
+ LockSuffix = ".lock"
+ DepSuffix = ".rec"
+ TmpPrefix = ".redo."
+ LogSuffix = ".log"
+ LogRecSuffix = ".log-rec"
)
-func mkdirs(pth string) error {
- if _, err := os.Stat(pth); err == nil {
- return nil
- }
- return os.MkdirAll(pth, os.FileMode(0777))
-}
+var (
+ NoSync = false
+ StderrKeep = false
+ StderrSilent = false
+ StderrPrefix string
+ StopIfMod = false
+ Jobs sync.WaitGroup
-func tempsuffix() string {
- return strconv.FormatInt((time.Now().UnixNano()+int64(os.Getpid()))&0xFFFFFFFF, 16)
-}
+ flagTrace *bool
+ flagTraceAll *bool
+ flagStderrKeep *bool
+ flagStderrSilent *bool
+ flagForcedIfchange *bool
-func tempfile(dir, prefix string) (*os.File, error) {
- // It respects umask, unlike ioutil.TempFile
- name := path.Join(dir, TmpPrefix+prefix+"."+tempsuffix())
- return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
-}
+ TracedAll bool
-func isModified(cwd, redoDir, tgt string) (bool, error) {
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
- if err != nil {
- if os.IsNotExist(err) {
- return false, nil
- }
- return false, err
- }
- defer fdDep.Close()
- r := recfile.NewReader(fdDep)
- for {
- m, err := r.NextMap()
- if err != nil {
- if err == io.EOF {
- break
- }
- return false, err
- }
- if m["Target"] != tgt {
- continue
- }
- fd, err := os.Open(path.Join(cwd, tgt))
- if err != nil {
- if os.IsNotExist(err) {
- return false, nil
- }
- return false, err
- }
- defer fd.Close()
- ourTs, err := fileCtime(fd)
- if err != nil {
- return false, err
- }
- if ourTs != m["Ctime"] {
- return true, nil
- }
- break
+ RunningProcs = map[int]*os.Process{}
+ RunningProcsM sync.Mutex
+
+ Err1WasTouched = errors.New("$1 was explicitly touched")
+)
+
+func init() {
+ cmdName := CmdName()
+ if !(cmdName == CmdNameRedo || cmdName == CmdNameRedoIfchange) {
+ return
}
- return false, nil
+ flagTrace = flag.Bool("x", false, "trace (sh -x) current targets")
+ flagTraceAll = flag.Bool("xx", false,
+ fmt.Sprintf("trace (sh -x) all targets (%s=1)", EnvTrace))
+ flagStderrKeep = flag.Bool("k", false,
+ fmt.Sprintf("keep job's stderr (%s=1)", EnvStderrKeep))
+ flagStderrSilent = flag.Bool("s", false,
+ fmt.Sprintf("silent, do not print job's stderr (%s=1)", EnvStderrSilent))
+ flagForcedIfchange = flag.Bool("f", false, "forced redo-ifchange")
}
-type RunErr struct {
- Tgt string
+type RunError struct {
+ Tgt *Tgt
DoFile string
Started *time.Time
Finished *time.Time
Err error
}
-func (e RunErr) Unwrap() error { return e.Err }
-
-func (e *RunErr) Name() string {
+func (e *RunError) Name() string {
var name string
if e.DoFile == "" {
- name = e.Tgt
+ name = e.Tgt.String()
} else {
name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile)
}
if e.Finished == nil {
return name
}
- return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds())
+ return fmt.Sprintf("%s (%.3fs)", name, e.Finished.Sub(*e.Started).Seconds())
}
-func (e RunErr) Error() string {
+func (e RunError) Error() string {
return fmt.Sprintf("%s: %s", e.Name(), e.Err)
}
+func mkdirs(pth string) error {
+ if FileExists(pth) {
+ return nil
+ }
+ return os.MkdirAll(pth, os.FileMode(0777))
+}
+
+func isModified(depInfo *DepInfo, tgt *Tgt) (
+ modified bool, ourInode *Inode, hshPrev Hash, err error,
+) {
+ if depInfo == nil {
+ return
+ }
+ for _, dep := range depInfo.ifchanges {
+ if dep.tgt.a != tgt.a {
+ continue
+ }
+ ourInode, err = inodeFromFileByPath(tgt.a)
+ if err != nil {
+ if os.IsNotExist(err) {
+ err = nil
+ return
+ }
+ err = ErrLine(err)
+ return
+ }
+ hshPrev = dep.hash
+ modified = !ourInode.Equals(dep.inode)
+ break
+ }
+ return
+}
+
func syncDir(dir string) error {
fd, err := os.Open(dir)
if err != nil {
return err
}
-func runScript(tgt string, errs chan error) error {
- tgtOrig := tgt
- cwd, tgt := cwdAndTgt(tgt)
- redoDir := path.Join(cwd, RedoDir)
+func runScript(tgt *Tgt, errs chan error, forced, traced bool) error {
+ redoDir := path.Join(tgt.h, RedoDir)
if err := mkdirs(redoDir); err != nil {
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
+ shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgt, tgt.h)
+ jsToken := jsAcquire(shCtx)
+ jsNeedsRelease := true
+ defer func() {
+ if jsNeedsRelease {
+ jsRelease(shCtx, jsToken)
+ }
+ }()
+
// Acquire lock
fdLock, err := os.OpenFile(
- path.Join(redoDir, tgt+LockSuffix),
+ path.Join(redoDir, tgt.t+LockSuffix),
os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
os.FileMode(0666),
)
if err != nil {
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
+ }
+ flock := unix.Flock_t{
+ Type: unix.F_WRLCK,
+ Whence: io.SeekStart,
}
lockRelease := func() {
- trace(CLock, "LOCK_UN: %s", fdLock.Name())
- unix.Flock(int(fdLock.Fd()), unix.LOCK_UN)
+ tracef(CLock, "LOCK_UN: %s", fdLock.Name())
+ flock.Type = unix.F_UNLCK
+ if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ log.Fatalln(err, fdLock.Name())
+ }
fdLock.Close()
}
- trace(CLock, "LOCK_NB: %s", fdLock.Name())
+ tracef(CLock, "LOCK_NB: %s", fdLock.Name())
// Waiting for job completion, already taken by someone else
- if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
- if uintptr(err.(syscall.Errno)) != uintptr(unix.EWOULDBLOCK) {
+ if err = unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ if uintptr(err.(syscall.Errno)) != uintptr(unix.EAGAIN) {
fdLock.Close()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
- trace(CDebug, "waiting: %s", tgtOrig)
Jobs.Add(1)
+ if err = unix.FcntlFlock(fdLock.Fd(), unix.F_GETLK, &flock); err != nil {
+ log.Fatalln(err, fdLock.Name())
+ }
+ tracef(CDebug, "waiting: %s (pid=%d)", tgt, flock.Pid)
+ if FdStatus != nil {
+ if _, err = FdStatus.Write([]byte{StatusWait}); err != nil {
+ log.Fatal(err)
+ }
+ }
go func() {
defer Jobs.Done()
- trace(CLock, "LOCK_EX: %s", fdLock.Name())
- unix.Flock(int(fdLock.Fd()), unix.LOCK_EX)
- lockRelease()
- trace(CDebug, "waiting done: %s", tgtOrig)
- var builtNow bool
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
- if err != nil {
- if os.IsNotExist(err) {
- err = errors.New("was not built")
+ tracef(CLock, "LOCK_EX: %s", fdLock.Name())
+ LockAgain:
+ if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLKW, &flock); err != nil {
+ if errors.Is(err, unix.EDEADLK) {
+ time.Sleep(10 * time.Millisecond)
+ goto LockAgain
}
- goto Finish
+ log.Fatalln(err, fdLock.Name())
}
- builtNow, _, err = isBuiltNow(fdDep)
- if err != nil {
- goto Finish
+ lockRelease()
+ tracef(CDebug, "waiting done: %s", tgt)
+ if FdStatus != nil {
+ if _, err = FdStatus.Write([]byte{StatusWaited}); err != nil {
+ log.Fatal(err)
+ }
}
- if !builtNow {
- err = errors.New("was not built")
+ build, err := depReadBuild(tgt.Dep())
+ if err == nil {
+ if build != BuildUUID {
+ err = errors.New("was not built: build differs")
+ }
+ } else {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = errors.New("was not built: no .rec")
+ }
}
- Finish:
if err != nil {
- err = TgtErr{tgtOrig, err}
+ err = TgtError{tgt, err}
}
errs <- err
}()
return nil
}
+ // It scans the whole .rec file while searching for the single target,
+ // but that one is always located at the very end
+ depInfo, err := depRead(tgt)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ lockRelease()
+ return TgtError{tgt, ErrLine(err)}
+ }
+ }
+
+ // Check if it was already built in parallel
+ if !forced && depInfo != nil && depInfo.build == BuildUUID {
+ lockRelease()
+ errs <- nil
+ return nil
+ }
+
// Check if target is not modified externally
- modified, err := isModified(cwd, redoDir, tgt)
+ modified, inodePrev, hshPrev, err := isModified(depInfo, tgt)
if err != nil {
lockRelease()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
if modified {
- trace(CWarn, "%s externally modified: not redoing", tgtOrig)
lockRelease()
- errs <- nil
- return TgtErr{tgtOrig, err}
+ if StopIfMod {
+ return fmt.Errorf("%s externally modified", tgt)
+ }
+ tracef(CWarn, "%s externally modified: not redoing", tgt)
+ go func() {
+ errs <- nil
+ }()
+ return nil
}
+ depInfo = nil
- // Start preparing .dep
- fdDep, err := tempfile(redoDir, tgt+DepSuffix)
+ // Start preparing .rec
+ fdDep, err := tempfile(redoDir, tgt.t+DepSuffix)
if err != nil {
lockRelease()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
+ fdDepOpened := true
+ fdDepExists := true
cleanup := func() {
lockRelease()
- fdDep.Close()
- os.Remove(fdDep.Name())
+ if fdDepOpened {
+ fdDep.Close()
+ }
+ if fdDepExists {
+ os.Remove(fdDep.Name())
+ }
}
if _, err = recfile.NewWriter(fdDep).WriteFields(
recfile.Field{Name: "Build", Value: BuildUUID},
); err != nil {
cleanup()
- return TgtErr{tgtOrig, err}
- }
-
- // Find .do
- doFile, upLevels, err := findDo(fdDep, cwd, tgt)
- if err != nil {
- cleanup()
- return TgtErr{tgtOrig, err}
- }
- if doFile == "" {
- cleanup()
- return TgtErr{tgtOrig, errors.New("no .do found")}
- }
- if err = writeDep(fdDep, cwd, doFile); err != nil {
- cleanup()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
+ var cwd string
+ var dirPrefix string
+ var doFile *Tgt
+ basename := tgt.t
+ runErr := RunError{Tgt: tgt}
// Determine basename and DIRPREFIX
- ents := strings.Split(cwd, "/")
- ents = ents[len(ents)-upLevels:]
- dirPrefix := path.Join(ents...)
- cwdOrig := cwd
- for i := 0; i < upLevels; i++ {
- cwd = path.Join(cwd, "..")
+ {
+ doFileRelPath, upLevels, err := findDo(fdDep, tgt.h, tgt.t)
+ if err != nil {
+ cleanup()
+ return TgtError{tgt, ErrLine(err)}
+ }
+ if doFileRelPath == "" {
+ cleanup()
+ return TgtError{tgt, errors.New("no .do found")}
+ }
+ // ents := strings.Split(strings.TrimSuffix(tgt.h, "/"), "/")
+ ents := strings.Split(tgt.h, "/")
+ ents = ents[len(ents)-upLevels:]
+ dirPrefix = path.Join(ents...)
+ ups := make([]string, 0, upLevels+2)
+ ups = append(ups, tgt.h)
+ for i := 0; i < upLevels; i++ {
+ ups = append(ups, "..")
+ }
+ ups = append(ups, doFileRelPath)
+ cwd = path.Clean(path.Join(ups[:len(ups)-1]...))
+ doFile = NewTgt(path.Join(ups...))
+ if strings.HasPrefix(doFile.t, "default.") {
+ basename = basename[:len(basename)-(len(doFile.t)-len("default.")-len(".do"))-1]
+ runErr.DoFile = doFile.String()
+ }
}
- cwd = path.Clean(cwd)
- basename := tgt
- runErr := RunErr{Tgt: tgtOrig}
- if strings.HasPrefix(doFile, "default.") {
- basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1]
- runErr.DoFile = doFile
+
+ if err = depWrite(fdDep, tgt.h, doFile, ""); err != nil {
+ cleanup()
+ return TgtError{tgt, ErrLine(err)}
}
- trace(CWait, "%s", runErr.Name())
- doFile = path.Base(doFile)
+ fdDep.Close()
+ fdDepOpened = false
+ tracef(CWait, "%s", runErr.Name())
// Prepare command line
var cmdName string
var args []string
- if err = unix.Access(path.Join(cwd, doFile), unix.X_OK); err == nil {
- // Ordinary executable file
- cmdName = doFile
+ if err = unix.Access(doFile.String(), unix.X_OK); err == nil {
+ cmdName = doFile.t
args = make([]string, 0, 3)
} else {
- fd, err := os.Open(path.Join(cwd, doFile))
- if err != nil {
- cleanup()
- return TgtErr{tgtOrig, err}
- }
- buf := make([]byte, 512)
- n, err := fd.Read(buf)
- if err != nil {
- cleanup()
- return TgtErr{tgtOrig, err}
- }
- if n > 3 && string(buf[:3]) == "#!/" {
- // Shebanged
- t := string(buf[2:n])
- nlIdx := strings.Index(t, "\n")
- if nlIdx == -1 {
- cleanup()
- return TgtErr{tgtOrig, errors.New("not fully read shebang")}
- }
- args = strings.Split(t[:nlIdx], " ")
- cmdName, args = args[0], args[1:]
+ cmdName = "/bin/sh"
+ if traced || TracedAll {
+ args = append(args, "-ex")
} else {
- // Shell
- cmdName = "/bin/sh"
- if Trace {
- args = append(args, "-ex")
- } else {
- args = append(args, "-e")
- }
+ args = append(args, "-e")
}
- args = append(args, doFile)
+ args = append(args, doFile.t)
}
// Temporary file for stdout
- fdStdout, err := tempfile(cwd, tgt)
+ fdStdout, err := tempfile(tgt.h, tgt.t)
if err != nil {
cleanup()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
+ fdStdout.Close()
tmpPath := fdStdout.Name() + ".3" // and for $3
- args = append(args, tgt, basename, path.Base(tmpPath))
+ tmpPathRel := mustRel(cwd, tmpPath)
+ args = append(
+ args,
+ path.Join(dirPrefix, tgt.t),
+ path.Join(dirPrefix, basename),
+ tmpPathRel,
+ )
+ shCtx = fmt.Sprintf(
+ "sh: %s: %s %s cwd:%s dirprefix:%s",
+ tgt, cmdName, args, cwd, dirPrefix,
+ )
cmd := exec.Command(cmdName, args...)
cmd.Dir = cwd
- cmd.Stdout = fdStdout
- cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", RedoLevelEnv, Level+1))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoDirPrefixEnv, dirPrefix))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoBuildUUIDEnv, BuildUUID))
- childStderrPrefix := tempsuffix()
+ // cmd.Stdin reads from /dev/null by default
+ cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix))
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID))
+
+ childStderrPrefixRaw := make([]byte, 8)
+ if _, err = io.ReadFull(rand.Reader, childStderrPrefixRaw); err != nil {
+ panic(err)
+ }
+ childStderrPrefix := hex.EncodeToString(childStderrPrefixRaw)
cmd.Env = append(cmd.Env, fmt.Sprintf(
- "%s=%s", RedoStderrPrefixEnv, childStderrPrefix,
+ "%s=%s", EnvStderrPrefix, childStderrPrefix,
))
- cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep)
fdNum := 0
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoDepFdEnv, 3+fdNum))
+ cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts)
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum))
+ fdNum++
+ cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock)
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum))
fdNum++
- if JSR == nil {
- // infinite jobs
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSRFdEnv))
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSWFdEnv))
+
+ if FdStatus == nil {
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd))
} else {
- cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
- cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSRFdEnv, 3+fdNum))
- fdNum++
- cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSWFdEnv, 3+fdNum))
+ cmd.ExtraFiles = append(cmd.ExtraFiles, FdStatus)
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvStatusFd, 3+fdNum))
fdNum++
}
// Preparing stderr
- stderr, err := cmd.StderrPipe()
- if err != nil {
- panic(err)
- }
var fdStderr *os.File
if StderrKeep {
fdStderr, err = os.OpenFile(
- path.Join(redoDir, tgt+LogSuffix),
- os.O_WRONLY|os.O_CREATE,
+ path.Join(redoDir, tgt.t+LogSuffix),
+ os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
os.FileMode(0666),
)
if err != nil {
cleanup()
- return TgtErr{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
- fdStderr.Truncate(0)
}
- shCtx := fmt.Sprintf("sh: %s: %s %s [%s]", tgtOrig, cmdName, args, cwd)
- trace(CDebug, "%s", shCtx)
+ tracef(CDebug, "%s", shCtx)
+ jsNeedsRelease = false
Jobs.Add(1)
go func() {
- jsAcquire(shCtx)
+ if JSR == nil {
+ // infinite jobs
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJobs))
+ } else {
+ cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
+ cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
+ makeFlags := fmt.Sprintf(
+ "%s %s%d,%d", MakeFlags, MakeJSArg, 3+fdNum+0, 3+fdNum+1,
+ )
+ makeFlags = strings.Trim(makeFlags, " ")
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", MakeFlagsName, makeFlags))
+ fdNum += 2
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvJSToken, jsToken))
+ }
+
+ if FdStatus != nil {
+ if _, err = FdStatus.Write([]byte{StatusRun}); err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ defer jsRelease(shCtx, jsToken)
+
+ var finished time.Time
+ var exitErr *exec.ExitError
+ started := time.Now()
+ runErr.Started = &started
+ fdStdout, err = os.OpenFile(fdStdout.Name(), os.O_RDWR, os.FileMode(0666))
+ if err != nil {
+ if fdStderr != nil {
+ fdStderr.Close()
+ }
+ runErr.Err = err
+ errs <- runErr
+ return
+ }
+ cmd.Stdout = fdStdout
+ fdDep, err = os.OpenFile(fdDep.Name(), os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
+ if err != nil {
+ if fdStderr != nil {
+ fdStderr.Close()
+ }
+ fdStdout.Close()
+ runErr.Err = err
+ errs <- runErr
+ return
+ }
+ fdDepOpened = true
+ cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep)
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum))
+ fdNum++
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDepCwd, cwd))
+
defer func() {
- jsRelease(shCtx)
- lockRelease()
- fdDep.Close()
+ if fdDepOpened {
+ fdDep.Close()
+ }
fdStdout.Close()
if fdStderr != nil {
fdStderr.Close()
+ logRecPath := path.Join(redoDir, tgt.t+LogRecSuffix)
+ if fdStderr, err = os.OpenFile(
+ logRecPath,
+ os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
+ os.FileMode(0666),
+ ); err == nil {
+ fields := []recfile.Field{
+ {Name: "Build", Value: BuildUUID},
+ {Name: "PPID", Value: strconv.Itoa(os.Getpid())},
+ {Name: "Cwd", Value: cwd},
+ }
+ if cmd.Process != nil {
+ fields = append(fields, recfile.Field{
+ Name: "PID", Value: strconv.Itoa(cmd.Process.Pid),
+ })
+ }
+ ts := new(tai64n.TAI64N)
+ ts.FromTime(started)
+ fields = append(fields,
+ recfile.Field{Name: "Started", Value: tai64n.Encode(ts[:])},
+ )
+ ts.FromTime(finished)
+ fields = append(fields,
+ recfile.Field{Name: "Finished", Value: tai64n.Encode(ts[:])})
+ fields = append(fields, recfile.Field{
+ Name: "Duration",
+ Value: strconv.FormatInt(finished.Sub(started).Nanoseconds(), 10),
+ })
+ fields = append(fields, recfile.Field{Name: "Cmd", Value: cmdName})
+ for _, arg := range args {
+ fields = append(fields, recfile.Field{Name: "Arg", Value: arg})
+ }
+ for _, env := range cmd.Env {
+ fields = append(fields, recfile.Field{Name: "Env", Value: env})
+ }
+ if exitErr != nil {
+ fields = append(fields, recfile.Field{
+ Name: "ExitCode",
+ Value: strconv.Itoa(exitErr.ProcessState.ExitCode()),
+ })
+ }
+
+ w := bufio.NewWriter(fdStderr)
+ {
+ var ifchanges []string
+ ifchanges, err = depReadOnlyIfchanges(tgt.Dep())
+ if err != nil {
+ err = ErrLine(err)
+ goto Err
+ }
+ for _, dep := range ifchanges {
+ fields = append(fields, recfile.Field{
+ Name: "Ifchange",
+ Value: dep,
+ })
+ }
+ }
+ _, err = recfile.NewWriter(w).WriteFields(fields...)
+ if err != nil {
+ err = ErrLine(err)
+ goto Err
+ }
+ err = ErrLine(w.Flush())
+ Err:
+ if err != nil {
+ log.Print(err)
+ os.Remove(logRecPath)
+ }
+ fdStderr.Close()
+ } else {
+ log.Println("can not open", logRecPath, ":", err)
+ }
+ }
+ lockRelease()
+ if fdDepExists {
+ os.Remove(fdDep.Name())
}
- os.Remove(fdDep.Name())
os.Remove(fdStdout.Name())
os.Remove(tmpPath)
- os.Remove(fdLock.Name())
+ if FdStatus != nil {
+ if _, err = FdStatus.Write([]byte{StatusDone}); err != nil {
+ log.Fatal(err)
+ }
+ }
Jobs.Done()
}()
- started := time.Now()
- runErr.Started = &started
- err := cmd.Start()
+ stderr, err := cmd.StderrPipe()
if err != nil {
runErr.Err = err
errs <- runErr
return
}
+ started = time.Now()
+ err = cmd.Start()
+ if err != nil {
+ runErr.Err = err
+ errs <- runErr
+ return
+ }
+ RunningProcsM.Lock()
+ RunningProcs[cmd.Process.Pid] = cmd.Process
+ RunningProcsM.Unlock()
pid := fmt.Sprintf("[%d]", cmd.Process.Pid)
- trace(CDebug, "%s runs %s", tgtOrig, pid)
+ tracef(CDebug, "%s runs %s", tgt, pid)
- stderrTerm := make(chan struct{}, 0)
+ stderrTerm := make(chan struct{})
go func() {
scanner := bufio.NewScanner(stderr)
var line string
- ts := new(TAI64N)
+ ts := new(tai64n.TAI64N)
for scanner.Scan() {
line = scanner.Text()
if strings.HasPrefix(line, childStderrPrefix) {
continue
}
if fdStderr != nil {
- tai64nNow(ts)
- fmt.Fprintf(fdStderr, "@%s %s\n", hex.EncodeToString(ts[:]), line)
+ ts.FromTime(time.Now())
+ LogMutex.Lock()
+ fmt.Fprintln(fdStderr, tai64n.Encode(ts[:]), line)
+ LogMutex.Unlock()
}
if StderrSilent {
continue
}
if MyPid == 0 {
- trace(CNone, "%s", line)
+ tracef(CNone, "%s", line)
} else {
- trace(CNone, "%s %s", pid, line)
+ tracef(CNone, "%s %s", pid, line)
}
}
close(stderrTerm)
// Wait for job completion
<-stderrTerm
err = cmd.Wait()
- finished := time.Now()
+ RunningProcsM.Lock()
+ delete(RunningProcs, cmd.Process.Pid)
+ RunningProcsM.Unlock()
+ finished = time.Now()
runErr.Finished = &finished
if err != nil {
+ exitErr = err.(*exec.ExitError)
runErr.Err = err
errs <- runErr
return
}
+ // Was $1 touched?
+ if inode, err := inodeFromFileByPath(tgt.a); err == nil {
+ if inodePrev == nil {
+ runErr.Err = Err1WasTouched
+ errs <- runErr
+ return
+ }
+ if err != nil {
+ runErr.Err = err
+ errs <- runErr
+ return
+ }
+ if !inode.Equals(inodePrev) {
+ runErr.Err = Err1WasTouched
+ errs <- runErr
+ return
+ }
+ }
+
// Does it produce both stdout and tmp?
fiStdout, err := os.Stat(fdStdout.Name())
if err != nil {
- runErr.Err = err
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
return
}
tmpExists = true
- } else if !os.IsNotExist(err) {
- runErr.Err = err
+ } else if !errors.Is(err, fs.ErrNotExist) {
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
if tmpExists {
fd, err = os.Open(tmpPath)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
defer fd.Close()
fd = fdStdout
}
- // Do we need to ifcreate it, of ifchange with renaming?
+ // Do we need to ifcreate it, or ifchange with renaming?
if fd == nil {
- err = ifcreate(fdDep, tgt)
+ os.Remove(tgt.a)
+ err = ifcreate(fdDep, tgt.t)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
} else {
+ var hsh Hash
+ if hshPrev != "" {
+ _, err = fd.Seek(0, io.SeekStart)
+ if err != nil {
+ err = ErrLine(err)
+ goto Finish
+ }
+ hsh, err = fileHash(fd)
+ if err != nil {
+ err = ErrLine(err)
+ goto Finish
+ }
+ if hsh == hshPrev {
+ tracef(CDebug, "%s has same hash, not renaming", tgt)
+ err = ErrLine(os.Remove(fd.Name()))
+ if err != nil {
+ goto Finish
+ }
+ err = ErrLine(os.Chtimes(tgt.a, finished, finished))
+ if err != nil {
+ goto Finish
+ }
+ if !NoSync {
+ err = ErrLine(syncDir(tgt.h))
+ if err != nil {
+ goto Finish
+ }
+ }
+ err = ErrLine(depWrite(fdDep, tgt.h, tgt, hsh))
+ if err != nil {
+ goto Finish
+ }
+ goto RecCommit
+ }
+ }
if !NoSync {
- err = fd.Sync()
+ err = ErrLine(fd.Sync())
if err != nil {
goto Finish
}
}
- err = os.Rename(fd.Name(), path.Join(cwdOrig, tgt))
+ err = ErrLine(os.Rename(fd.Name(), tgt.a))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(cwdOrig)
+ err = ErrLine(syncDir(tgt.h))
if err != nil {
goto Finish
}
}
- err = writeDep(fdDep, cwdOrig, tgt)
+ err = ErrLine(depWrite(fdDep, tgt.h, tgt, hsh))
if err != nil {
goto Finish
}
}
- // Commit .dep
+ RecCommit:
+ // Commit .rec
if !NoSync {
- err = fdDep.Sync()
+ err = ErrLine(fdDep.Sync())
if err != nil {
goto Finish
}
}
- err = os.Rename(fdDep.Name(), path.Join(redoDir, tgt+DepSuffix))
+ err = ErrLine(os.Rename(fdDep.Name(), tgt.Dep()))
if err != nil {
goto Finish
}
+ fdDepExists = false
if !NoSync {
- err = syncDir(redoDir)
+ err = ErrLine(syncDir(redoDir))
if err != nil {
goto Finish
}
}
+ fdDep.Close()
+ fdDepOpened = false
+
+ // Post-commit .rec sanitizing
+ if depInfo, err := depRead(tgt); err == nil {
+ ifchangeSeen := make(map[string]struct{}, len(depInfo.ifchanges))
+ for _, dep := range depInfo.ifchanges {
+ ifchangeSeen[dep.tgt.a] = struct{}{}
+ }
+ for _, dep := range depInfo.ifcreates {
+ if _, exists := ifchangeSeen[dep.a]; exists {
+ tracef(CWarn, "simultaneous ifcreate and ifchange records: %s", tgt)
+ }
+ }
+ } else if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ err = ErrLine(err)
+ goto Finish
+ }
+
Finish:
runErr.Err = err
errs <- runErr
}()
return nil
}
+
+func isOkRun(err error) bool {
+ if err == nil {
+ return true
+ }
+ var runErr RunError
+ if errors.As(err, &runErr) && runErr.Err == nil {
+ tracef(CRedo, "%s", runErr.Name())
+ return true
+ }
+ tracef(CErr, "%s", err)
+ return false
+}