"flag"
"fmt"
"io"
+ "io/fs"
"log"
"os"
"os/exec"
RedoDir = ".redo"
LockSuffix = ".lock"
- DepSuffix = ".rec"
+ DepSuffix = ".dep"
TmpPrefix = ".redo."
LogSuffix = ".log"
LogRecSuffix = ".log-rec"
}
type RunError struct {
- Tgt string
+ Tgt *Tgt
DoFile string
Started *time.Time
Finished *time.Time
func (e *RunError) Name() string {
var name string
if e.DoFile == "" {
- name = e.Tgt
+ name = e.Tgt.rel
} else {
name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile)
}
return os.MkdirAll(pth, os.FileMode(0777))
}
-func isModified(cwd, redoDir, tgt string) (bool, *Inode, string, error) {
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
- if err != nil {
- if os.IsNotExist(err) {
- return false, nil, "", nil
- }
- return false, nil, "", ErrLine(err)
+func isModified(dep *Dep, tgt *Tgt) (
+ modified bool, ourInode *Inode, hshPrev Hash, err error,
+) {
+ if dep == nil {
+ return
}
- defer fdDep.Close()
- r := recfile.NewReader(fdDep)
- var modified bool
- var ourInode *Inode
- var hshPrev string
- for {
- m, err := r.NextMap()
- if err != nil {
- if errors.Is(err, io.EOF) {
- break
- }
- return false, nil, "", ErrLine(err)
- }
- if m["Type"] != DepTypeIfchange || m["Target"] != tgt {
+ for _, ifchange := range dep.ifchanges {
+ if ifchange.tgt.a != tgt.a {
continue
}
- ourInode, err = inodeFromFileByPath(path.Join(cwd, tgt))
+ ourInode, err = inodeFromFileByPath(tgt.a)
if err != nil {
if os.IsNotExist(err) {
- return false, nil, "", nil
+ err = nil
+ return
}
- return false, nil, "", ErrLine(err)
- }
- theirInode, err := inodeFromRec(m)
- if err != nil {
- return false, nil, "", ErrLine(err)
+ err = ErrLine(err)
+ return
}
- hshPrev = m["Hash"]
- modified = !ourInode.Equals(theirInode)
+ hshPrev = ifchange.Hash()
+ modified = !ourInode.Equals(ifchange.Inode())
break
}
- return modified, ourInode, hshPrev, nil
+ return
}
func syncDir(dir string) error {
return err
}
-func runScript(tgtOrig string, errs chan error, forced, traced bool) error {
- cwd, tgt := cwdAndTgt(tgtOrig)
- redoDir := path.Join(cwd, RedoDir)
+func runScript(tgt *Tgt, errs chan error, forced, traced bool) error {
+ tgtH, tgtT := pathSplit(tgt.a)
+ redoDir := path.Join(tgtH, RedoDir)
if err := mkdirs(redoDir); err != nil {
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
- shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgtOrig, cwd)
+ shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgt, tgtH)
jsToken := jsAcquire(shCtx)
jsNeedsRelease := true
defer func() {
// Acquire lock
fdLock, err := os.OpenFile(
- path.Join(redoDir, tgt+LockSuffix),
+ path.Join(redoDir, tgtT+LockSuffix),
os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
os.FileMode(0666),
)
if err != nil {
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
flock := unix.Flock_t{
Type: unix.F_WRLCK,
if err = unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
if uintptr(err.(syscall.Errno)) != uintptr(unix.EAGAIN) {
fdLock.Close()
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
Jobs.Add(1)
if err = unix.FcntlFlock(fdLock.Fd(), unix.F_GETLK, &flock); err != nil {
log.Fatalln(err, fdLock.Name())
}
- tracef(CDebug, "waiting: %s (pid=%d)", tgtOrig, flock.Pid)
+ tracef(CDebug, "waiting: %s (pid=%d)", tgt, flock.Pid)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWait}); err != nil {
log.Fatal(err)
log.Fatalln(err, fdLock.Name())
}
lockRelease()
- tracef(CDebug, "waiting done: %s", tgtOrig)
+ tracef(CDebug, "waiting done: %s", tgt)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWaited}); err != nil {
log.Fatal(err)
}
}
- var depInfo *DepInfo
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
- if err != nil {
+ build, err := depBuildRead(tgt.dep)
+ if err == nil {
+ if build != BuildUUID {
+ err = errors.New("was not built: build differs")
+ }
+ } else {
if errors.Is(err, fs.ErrNotExist) {
- err = errors.New("was not built: no .rec")
+ err = errors.New("was not built: no .dep")
}
- goto Finish
- }
- defer fdDep.Close()
- depInfo, err = depRead(fdDep)
- if err != nil {
- goto Finish
- }
- if depInfo.build != BuildUUID {
- err = errors.New("was not built: build differs")
}
- Finish:
if err != nil {
- err = TgtError{tgtOrig, err}
- fdLock.Close()
+ err = TgtError{tgt, err}
}
errs <- err
}()
return nil
}
- // Check if it was already built in parallel
- if !forced {
- if fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)); err == nil {
- depInfo, err := depRead(fdDep)
- fdDep.Close()
- if err == nil && depInfo.build == BuildUUID {
- lockRelease()
- errs <- nil
- return nil
- }
+ // It scans the whole .dep file while searching for the single target,
+ // but that one is always located at the very end
+ dep, err := depRead(tgt)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ lockRelease()
+ return TgtError{tgt, ErrLine(err)}
}
}
+ // Check if it was already built in parallel
+ if !forced && dep != nil && dep.build == BuildUUID {
+ lockRelease()
+ errs <- nil
+ return nil
+ }
+
// Check if target is not modified externally
- modified, inodePrev, hshPrev, err := isModified(cwd, redoDir, tgt)
+ modified, inodePrev, hshPrev, err := isModified(dep, tgt)
if err != nil {
lockRelease()
- return TgtError{tgtOrig, err}
+ return TgtError{tgt, ErrLine(err)}
}
if modified {
lockRelease()
if StopIfMod {
- return fmt.Errorf("%s externally modified", tgtOrig)
+ return fmt.Errorf("%s externally modified", tgt)
}
- tracef(CWarn, "%s externally modified: not redoing", tgtOrig)
+ tracef(CWarn, "%s externally modified: not redoing", tgt)
go func() {
errs <- nil
}()
return nil
}
+ dep = nil
- // Start preparing .rec
- fdDep, err := tempfile(redoDir, tgt+DepSuffix)
+ // Start preparing .dep
+ fdDep, err := tempfile(redoDir, tgtT+DepSuffix)
if err != nil {
lockRelease()
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
- fdDepPath := fdDep.Name()
+ fdDepOpened := true
+ fdDepExists := true
+ fdDepW := bufio.NewWriter(fdDep)
cleanup := func() {
lockRelease()
- fdDep.Close()
- os.Remove(fdDep.Name())
- }
- if _, err = recfile.NewWriter(fdDep).WriteFields(
- recfile.Field{Name: "Build", Value: BuildUUID},
- ); err != nil {
- cleanup()
- return TgtError{tgtOrig, ErrLine(err)}
- }
-
- // Find .do
- doFile, upLevels, err := findDo(fdDep, cwd, tgt)
- if err != nil {
- cleanup()
- return TgtError{tgtOrig, ErrLine(err)}
+ if fdDepOpened {
+ fdDep.Close()
+ }
+ if fdDepExists {
+ os.Remove(fdDep.Name())
+ }
}
- if doFile == "" {
+ if err = depBuildWrite(fdDepW, BuildUUID); err != nil {
cleanup()
- return TgtError{tgtOrig, errors.New("no .do found")}
+ return TgtError{tgt, ErrLine(err)}
}
+ var cwd string
+ var dirPrefix string
+ var doFile *Tgt
+ var doFileT string
+ basename := tgtT
+ runErr := RunError{Tgt: tgt}
// Determine basename and DIRPREFIX
- doFileRelPath := doFile
- ents := strings.Split(cwd, "/")
- ents = ents[len(ents)-upLevels:]
- dirPrefix := path.Join(ents...)
- cwdOrig := cwd
- for i := 0; i < upLevels; i++ {
- cwd = path.Join(cwd, "..")
- doFileRelPath = path.Join("..", doFileRelPath)
- }
- cwd = path.Clean(cwd)
- doFilePath := path.Join(cwd, doFile)
- basename := tgt
- runErr := RunError{Tgt: tgtOrig}
- if strings.HasPrefix(doFile, "default.") {
- basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1]
- runErr.DoFile = doFileRelPath
+ {
+ doFileRelPath, upLevels, err := findDo(fdDepW, fdDep.Name(), tgtH, tgtT)
+ if err != nil {
+ cleanup()
+ return TgtError{tgt, ErrLine(err)}
+ }
+ if doFileRelPath == "" {
+ cleanup()
+ return TgtError{tgt, errors.New("no .do found")}
+ }
+ ents := strings.Split(tgtH, "/")
+ ents = ents[len(ents)-upLevels:]
+ dirPrefix = path.Join(ents...)
+ ups := make([]string, 0, upLevels+2)
+ ups = append(ups, tgtH)
+ for i := 0; i < upLevels; i++ {
+ ups = append(ups, "..")
+ }
+ ups = append(ups, doFileRelPath)
+ cwd = path.Clean(path.Join(ups[:len(ups)-1]...))
+ doFile = NewTgt(path.Join(ups...))
+ doFileT = path.Base(doFile.a)
+ if strings.HasPrefix(doFileT, "default.") {
+ basename = basename[:len(basename)-(len(doFileT)-len("default.")-len(".do"))-1]
+ runErr.DoFile = doFile.rel
+ }
}
- if err = depWrite(fdDep, cwdOrig, doFileRelPath, ""); err != nil {
+ if err = depWrite(fdDepW, fdDep.Name(), tgtH, doFile, ""); err != nil {
+ cleanup()
+ return TgtError{tgt, ErrLine(err)}
+ }
+ if err = fdDepW.Flush(); err != nil {
cleanup()
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
fdDep.Close()
+ fdDepOpened = false
tracef(CWait, "%s", runErr.Name())
// Prepare command line
var cmdName string
var args []string
- if err = unix.Access(doFilePath, unix.X_OK); err == nil {
- cmdName = doFilePath
+ if err = unix.Access(doFile.rel, unix.X_OK); err == nil {
+ cmdName = doFile.a
args = make([]string, 0, 3)
} else {
cmdName = "/bin/sh"
} else {
args = append(args, "-e")
}
- args = append(args, doFile)
+ args = append(args, doFileT)
}
// Temporary file for stdout
- fdStdout, err := tempfile(cwdOrig, tgt)
+ fdStdout, err := tempfile(tgtH, tgtT)
if err != nil {
cleanup()
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
- stdoutPath := fdStdout.Name()
fdStdout.Close()
- tmpPath := stdoutPath + ".3" // and for $3
+ tmpPath := fdStdout.Name() + ".3" // and for $3
tmpPathRel := mustRel(cwd, tmpPath)
args = append(
args,
- path.Join(dirPrefix, tgt),
+ path.Join(dirPrefix, tgtT),
path.Join(dirPrefix, basename),
tmpPathRel,
)
shCtx = fmt.Sprintf(
"sh: %s: %s %s cwd:%s dirprefix:%s",
- tgtOrig, cmdName, args, cwd, dirPrefix,
+ tgt, cmdName, args, cwd, dirPrefix,
)
cmd := exec.Command(cmdName, args...)
var fdStderr *os.File
if StderrKeep {
fdStderr, err = os.OpenFile(
- path.Join(redoDir, tgt+LogSuffix),
+ path.Join(redoDir, tgtT+LogSuffix),
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
os.FileMode(0666),
)
if err != nil {
cleanup()
- return TgtError{tgtOrig, ErrLine(err)}
+ return TgtError{tgt, ErrLine(err)}
}
}
tracef(CDebug, "%s", shCtx)
var exitErr *exec.ExitError
started := time.Now()
runErr.Started = &started
- fdStdout, err = os.OpenFile(stdoutPath, os.O_RDWR, os.FileMode(0666))
+ fdStdout, err = os.OpenFile(fdStdout.Name(), os.O_RDWR, os.FileMode(0666))
if err != nil {
if fdStderr != nil {
fdStderr.Close()
return
}
cmd.Stdout = fdStdout
- fdDep, err = os.OpenFile(fdDepPath, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
+ fdDep, err = os.OpenFile(fdDep.Name(), os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
if err != nil {
if fdStderr != nil {
fdStderr.Close()
errs <- runErr
return
}
+ fdDepOpened = true
+ fdDepW.Reset(fdDep)
cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep)
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum))
fdNum++
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDepCwd, cwd))
defer func() {
- fdDep.Close()
+ if fdDepOpened {
+ fdDep.Close()
+ }
fdStdout.Close()
if fdStderr != nil {
fdStderr.Close()
- logRecPath := path.Join(redoDir, tgt+LogRecSuffix)
+ logRecPath := path.Join(redoDir, tgtT+LogRecSuffix)
if fdStderr, err = os.OpenFile(
logRecPath,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC,
os.FileMode(0666),
); err == nil {
fields := []recfile.Field{
- {Name: "Build", Value: BuildUUID},
+ {Name: "Build", Value: BuildUUID.String()},
{Name: "PPID", Value: strconv.Itoa(os.Getpid())},
{Name: "Cwd", Value: cwd},
}
Value: strconv.Itoa(exitErr.ProcessState.ExitCode()),
})
}
- w := bufio.NewWriter(fdStderr)
- var depInfo *DepInfo
- fdDep, err := os.Open(fdDepPath)
- if err != nil {
- err = ErrLine(err)
- goto Err
- }
- depInfo, err = depRead(fdDep)
- fdDep.Close()
- if err != nil {
- err = ErrLine(err)
- goto Err
- }
- for _, dep := range depInfo.ifchanges {
- fields = append(fields, recfile.Field{
- Name: "Ifchange",
- Value: dep["Target"],
- })
+ w := bufio.NewWriter(fdStderr)
+ {
+ var ifchanges []string
+ ifchanges, err = depReadOnlyIfchanges(tgt.dep)
+ if err != nil {
+ err = ErrLine(err)
+ goto Err
+ }
+ for _, ifchange := range ifchanges {
+ fields = append(fields, recfile.Field{
+ Name: "Ifchange",
+ Value: ifchange,
+ })
+ }
}
_, err = recfile.NewWriter(w).WriteFields(fields...)
if err != nil {
}
}
lockRelease()
- os.Remove(fdDep.Name())
+ if fdDepExists {
+ os.Remove(fdDep.Name())
+ }
os.Remove(fdStdout.Name())
os.Remove(tmpPath)
if FdStatus != nil {
RunningProcs[cmd.Process.Pid] = cmd.Process
RunningProcsM.Unlock()
pid := fmt.Sprintf("[%d]", cmd.Process.Pid)
- tracef(CDebug, "%s runs %s", tgtOrig, pid)
+ tracef(CDebug, "%s runs %s", tgt, pid)
stderrTerm := make(chan struct{})
go func() {
line = scanner.Text()
if strings.HasPrefix(line, childStderrPrefix) {
line = line[len(childStderrPrefix):]
- os.Stderr.WriteString(StderrPrefix + line + "\n")
+ if StderrPrefix == "" {
+ withPrependedTgt(line)
+ } else {
+ stderrWrite(StderrPrefix + line + "\n")
+ }
continue
}
if fdStderr != nil {
ts.FromTime(time.Now())
- LogMutex.Lock()
fmt.Fprintln(fdStderr, tai64n.Encode(ts[:]), line)
- LogMutex.Unlock()
}
if StderrSilent {
continue
}
- if MyPid == 0 {
- tracef(CNone, "%s", line)
+ if MyPID != 0 {
+ line = pid + " " + line
+ }
+ if StderrPrefix == "" {
+ withPrependedTgt("[" + tgt.rel + "]" + line)
} else {
- tracef(CNone, "%s %s", pid, line)
+ stderrWrite(StderrPrefix + "[" + tgt.rel + "]" + line + "\n")
}
}
close(stderrTerm)
}
// Was $1 touched?
- if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil {
+ if inode, err := inodeFromFileByPath(tgt.a); err == nil {
if inodePrev == nil {
runErr.Err = Err1WasTouched
errs <- runErr
return
}
- if err != nil {
- runErr.Err = err
- errs <- runErr
- return
- }
if !inode.Equals(inodePrev) {
runErr.Err = Err1WasTouched
errs <- runErr
}
}
- if inodePrev != nil {
- if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil {
- if !inode.Equals(inodePrev) {
- runErr.Err = Err1WasTouched
- errs <- runErr
- return
- }
- }
- }
-
// Does it produce both stdout and tmp?
fiStdout, err := os.Stat(fdStdout.Name())
if err != nil {
// Determine what file we must process at last
var fd *os.File
+ var chmod fs.FileMode
if tmpExists {
fd, err = os.Open(tmpPath)
if err != nil {
goto Finish
}
defer fd.Close()
+ if fi, rerr := fd.Stat(); rerr == nil {
+ chmod = fi.Mode()
+ } else {
+ err = ErrLine(rerr)
+ goto Finish
+ }
} else if fiStdout.Size() > 0 {
fd = fdStdout
}
// Do we need to ifcreate it, or ifchange with renaming?
if fd == nil {
- os.Remove(path.Join(cwdOrig, tgt))
- err = ifcreate(fdDep, tgt)
+ os.Remove(tgt.a)
+ err = ifcreate(fdDepW, fdDep.Name(), tgtT)
if err != nil {
err = ErrLine(err)
goto Finish
}
} else {
- var hsh string
+ var hsh Hash
if hshPrev != "" {
_, err = fd.Seek(0, io.SeekStart)
if err != nil {
goto Finish
}
if hsh == hshPrev {
- tracef(CDebug, "%s has same hash, not renaming", tgtOrig)
+ tracef(CDebug, "%s has same hash, not renaming", tgt)
err = ErrLine(os.Remove(fd.Name()))
if err != nil {
goto Finish
}
- err = ErrLine(os.Chtimes(path.Join(cwdOrig, tgt), finished, finished))
+ if chmod != 0 {
+ err = ErrLine(os.Chmod(tgt.a, chmod))
+ if err != nil {
+ goto Finish
+ }
+ }
+ err = ErrLine(os.Chtimes(tgt.a, finished, finished))
if err != nil {
goto Finish
}
if !NoSync {
- err = ErrLine(syncDir(cwdOrig))
+ err = ErrLine(syncDir(tgtH))
if err != nil {
goto Finish
}
}
- err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hshPrev))
+ err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtH, tgt, hsh))
if err != nil {
goto Finish
}
goto Finish
}
}
- err = ErrLine(os.Rename(fd.Name(), path.Join(cwdOrig, tgt)))
+ err = ErrLine(os.Rename(fd.Name(), tgt.a))
if err != nil {
goto Finish
}
if !NoSync {
- err = ErrLine(syncDir(cwdOrig))
+ err = ErrLine(syncDir(tgtH))
if err != nil {
goto Finish
}
}
- err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh))
+ err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtH, tgt, hsh))
if err != nil {
goto Finish
}
}
RecCommit:
- // Commit .rec
+ // Commit .dep
+ err = ErrLine(fdDepW.Flush())
+ if err != nil {
+ goto Finish
+ }
if !NoSync {
err = ErrLine(fdDep.Sync())
if err != nil {
goto Finish
}
}
- fdDepPath = path.Join(redoDir, tgt+DepSuffix)
- err = ErrLine(os.Rename(fdDep.Name(), fdDepPath))
+ err = ErrLine(os.Rename(fdDep.Name(), tgt.dep))
if err != nil {
goto Finish
}
+ fdDepExists = false
if !NoSync {
err = ErrLine(syncDir(redoDir))
if err != nil {
goto Finish
}
}
-
- // Post-commit .rec sanitizing
fdDep.Close()
- if fdDepR, err := os.Open(fdDepPath); err == nil {
- depInfo, err := depRead(fdDepR)
- fdDepR.Close()
- if err != nil {
- err = ErrLine(err)
- goto Finish
- }
- ifchangeSeen := make(map[string]struct{}, len(depInfo.ifchanges))
- for _, dep := range depInfo.ifchanges {
- ifchangeSeen[dep["Target"]] = struct{}{}
+ fdDepOpened = false
+
+ // Post-commit .dep sanitizing
+ dep, err = depRead(tgt)
+ if err == nil {
+ ifchangeSeen := make(map[string]struct{}, len(dep.ifchanges))
+ for _, ifchange := range dep.ifchanges {
+ ifchangeSeen[ifchange.tgt.a] = struct{}{}
}
- for _, dep := range depInfo.ifcreates {
- if _, exists := ifchangeSeen[dep]; exists {
+ for _, ifcreate := range dep.ifcreates {
+ if _, exists := ifchangeSeen[ifcreate.a]; exists {
tracef(CWarn, "simultaneous ifcreate and ifchange records: %s", tgt)
}
}
+ } else if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ err = ErrLine(err)
+ goto Finish
}
Finish: