/*
goredo -- djb's redo implementation on pure Go
-Copyright (C) 2020-2022 Sergey Matveev <stargrave@stargrave.org>
+Copyright (C) 2020-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
import (
"bufio"
+ "crypto/rand"
+ "encoding/hex"
"errors"
"flag"
"fmt"
EnvStderrKeep = "REDO_LOGS"
EnvStderrSilent = "REDO_SILENT"
EnvNoSync = "REDO_NO_SYNC"
+ EnvStopIfMod = "REDO_STOP_IF_MODIFIED"
RedoDir = ".redo"
LockSuffix = ".lock"
StderrKeep = false
StderrSilent = false
StderrPrefix string
+ StopIfMod = false
Jobs sync.WaitGroup
- flagTrace *bool
- flagTraceAll *bool
- flagStderrKeep *bool
- flagStderrSilent *bool
+ flagTrace *bool
+ flagTraceAll *bool
+ flagStderrKeep *bool
+ flagStderrSilent *bool
+ flagForcedIfchange *bool
TracedAll bool
fmt.Sprintf("keep job's stderr (%s=1)", EnvStderrKeep))
flagStderrSilent = flag.Bool("s", false,
fmt.Sprintf("silent, do not print job's stderr (%s=1)", EnvStderrSilent))
+ flagForcedIfchange = flag.Bool("f", false, "forced redo-ifchange")
}
type RunError struct {
}
func mkdirs(pth string) error {
- if _, err := os.Stat(pth); err == nil {
+ if FileExists(pth) {
return nil
}
return os.MkdirAll(pth, os.FileMode(0777))
}
-func isModified(cwd, redoDir, tgt string) (bool, *Inode, error) {
+func isModified(cwd, redoDir, tgt string) (bool, *Inode, string, error) {
fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
if err != nil {
if os.IsNotExist(err) {
- return false, nil, nil
+ return false, nil, "", nil
}
- return false, nil, err
+ return false, nil, "", ErrLine(err)
}
defer fdDep.Close()
r := recfile.NewReader(fdDep)
+ var modified bool
var ourInode *Inode
+ var hshPrev string
for {
m, err := r.NextMap()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
- return false, nil, err
+ return false, nil, "", ErrLine(err)
}
if m["Type"] != DepTypeIfchange || m["Target"] != tgt {
continue
}
- fd, err := os.Open(path.Join(cwd, tgt))
+ ourInode, err = inodeFromFileByPath(path.Join(cwd, tgt))
if err != nil {
if os.IsNotExist(err) {
- return false, nil, nil
+ return false, nil, "", nil
}
- return false, nil, err
- }
- ourInode, err = inodeFromFile(fd)
- fd.Close()
- if err != nil {
- return false, nil, err
+ return false, nil, "", ErrLine(err)
}
theirInode, err := inodeFromRec(m)
if err != nil {
- return false, nil, err
- }
- if !ourInode.Equals(theirInode) {
- return true, ourInode, nil
+ return false, nil, "", ErrLine(err)
}
+ hshPrev = m["Hash"]
+ modified = !ourInode.Equals(theirInode)
break
}
- return false, ourInode, nil
+ return modified, ourInode, hshPrev, nil
}
func syncDir(dir string) error {
return err
}
-func runScript(tgtOrig string, errs chan error, traced bool) error {
+func runScript(tgtOrig string, errs chan error, forced, traced bool) error {
cwd, tgt := cwdAndTgt(tgtOrig)
redoDir := path.Join(cwd, RedoDir)
if err := mkdirs(redoDir); err != nil {
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
+ shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgtOrig, cwd)
+ jsToken := jsAcquire(shCtx)
+ jsNeedsRelease := true
+ defer func() {
+ if jsNeedsRelease {
+ jsRelease(shCtx, jsToken)
+ }
+ }()
+
// Acquire lock
fdLock, err := os.OpenFile(
path.Join(redoDir, tgt+LockSuffix),
os.FileMode(0666),
)
if err != nil {
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
+ }
+ flock := unix.Flock_t{
+ Type: unix.F_WRLCK,
+ Whence: io.SeekStart,
}
lockRelease := func() {
tracef(CLock, "LOCK_UN: %s", fdLock.Name())
- if err := unix.Flock(int(fdLock.Fd()), unix.LOCK_UN); err != nil {
- log.Fatalln(err)
+ flock.Type = unix.F_UNLCK
+ if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ log.Fatalln(err, fdLock.Name())
}
fdLock.Close()
}
tracef(CLock, "LOCK_NB: %s", fdLock.Name())
// Waiting for job completion, already taken by someone else
- if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
- if uintptr(err.(syscall.Errno)) != uintptr(unix.EWOULDBLOCK) {
+ if err = unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ if uintptr(err.(syscall.Errno)) != uintptr(unix.EAGAIN) {
fdLock.Close()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
Jobs.Add(1)
- tracef(CDebug, "waiting: %s", tgtOrig)
+ if err = unix.FcntlFlock(fdLock.Fd(), unix.F_GETLK, &flock); err != nil {
+ log.Fatalln(err, fdLock.Name())
+ }
+ tracef(CDebug, "waiting: %s (pid=%d)", tgtOrig, flock.Pid)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWait}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
go func() {
defer Jobs.Done()
tracef(CLock, "LOCK_EX: %s", fdLock.Name())
- if err := unix.Flock(int(fdLock.Fd()), unix.LOCK_EX); err != nil {
- log.Fatalln(err)
+ LockAgain:
+ if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLKW, &flock); err != nil {
+ if errors.Is(err, unix.EDEADLK) {
+ time.Sleep(10 * time.Millisecond)
+ goto LockAgain
+ }
+ log.Fatalln(err, fdLock.Name())
}
lockRelease()
tracef(CDebug, "waiting done: %s", tgtOrig)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWaited}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
var depInfo *DepInfo
Finish:
if err != nil {
err = TgtError{tgtOrig, err}
+ fdLock.Close()
}
errs <- err
}()
return nil
}
+ // Check if it was already built in parallel
+ if !forced {
+ if fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)); err == nil {
+ depInfo, err := depRead(fdDep)
+ fdDep.Close()
+ if err == nil && depInfo.build == BuildUUID {
+ lockRelease()
+ errs <- nil
+ return nil
+ }
+ }
+ }
+
// Check if target is not modified externally
- modified, inodePrev, err := isModified(cwd, redoDir, tgt)
+ modified, inodePrev, hshPrev, err := isModified(cwd, redoDir, tgt)
if err != nil {
lockRelease()
return TgtError{tgtOrig, err}
}
if modified {
- tracef(CWarn, "%s externally modified: not redoing", tgtOrig)
lockRelease()
+ if StopIfMod {
+ return fmt.Errorf("%s externally modified", tgtOrig)
+ }
+ tracef(CWarn, "%s externally modified: not redoing", tgtOrig)
go func() {
errs <- nil
}()
fdDep, err := tempfile(redoDir, tgt+DepSuffix)
if err != nil {
lockRelease()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
fdDepPath := fdDep.Name()
cleanup := func() {
recfile.Field{Name: "Build", Value: BuildUUID},
); err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
// Find .do
doFile, upLevels, err := findDo(fdDep, cwd, tgt)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
if doFile == "" {
cleanup()
runErr.DoFile = doFileRelPath
}
- if err = depWrite(fdDep, cwdOrig, doFileRelPath); err != nil {
+ if err = depWrite(fdDep, cwdOrig, doFileRelPath, ""); err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
fdDep.Close()
tracef(CWait, "%s", runErr.Name())
fdStdout, err := tempfile(cwdOrig, tgt)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
stdoutPath := fdStdout.Name()
fdStdout.Close()
path.Join(dirPrefix, basename),
tmpPathRel,
)
+ shCtx = fmt.Sprintf(
+ "sh: %s: %s %s cwd:%s dirprefix:%s",
+ tgtOrig, cmdName, args, cwd, dirPrefix,
+ )
cmd := exec.Command(cmdName, args...)
cmd.Dir = cwd
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix))
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID))
- childStderrPrefix := tempsuffix()
+ childStderrPrefixRaw := make([]byte, 8)
+ if _, err = io.ReadFull(rand.Reader, childStderrPrefixRaw); err != nil {
+ panic(err)
+ }
+ childStderrPrefix := hex.EncodeToString(childStderrPrefixRaw)
cmd.Env = append(cmd.Env, fmt.Sprintf(
"%s=%s", EnvStderrPrefix, childStderrPrefix,
))
)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
}
- shCtx := fmt.Sprintf(
- "sh: %s: %s %s cwd:%s dirprefix:%s",
- tgtOrig, cmdName, args, cwd, dirPrefix,
- )
tracef(CDebug, "%s", shCtx)
+ jsNeedsRelease = false
Jobs.Add(1)
go func() {
- jsToken := jsAcquire(shCtx)
if JSR == nil {
// infinite jobs
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJobs))
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusRun}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
+ defer jsRelease(shCtx, jsToken)
+
var finished time.Time
var exitErr *exec.ExitError
started := time.Now()
runErr.Started = &started
fdStdout, err = os.OpenFile(stdoutPath, os.O_RDWR, os.FileMode(0666))
if err != nil {
+ if fdStderr != nil {
+ fdStderr.Close()
+ }
runErr.Err = err
errs <- runErr
return
cmd.Stdout = fdStdout
fdDep, err = os.OpenFile(fdDepPath, os.O_WRONLY|os.O_APPEND, os.FileMode(0666))
if err != nil {
+ if fdStderr != nil {
+ fdStderr.Close()
+ }
+ fdStdout.Close()
runErr.Err = err
errs <- runErr
return
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDepCwd, cwd))
defer func() {
- jsRelease(shCtx, jsToken)
fdDep.Close()
fdStdout.Close()
if fdStderr != nil {
var depInfo *DepInfo
fdDep, err := os.Open(fdDepPath)
if err != nil {
+ err = ErrLine(err)
goto Err
}
depInfo, err = depRead(fdDep)
fdDep.Close()
if err != nil {
+ err = ErrLine(err)
goto Err
}
for _, dep := range depInfo.ifchanges {
}
_, err = recfile.NewWriter(w).WriteFields(fields...)
if err != nil {
+ err = ErrLine(err)
goto Err
}
- err = w.Flush()
+ err = ErrLine(w.Flush())
Err:
if err != nil {
- log.Println(err)
+ log.Print(err)
os.Remove(logRecPath)
}
fdStderr.Close()
os.Remove(fdDep.Name())
os.Remove(fdStdout.Name())
os.Remove(tmpPath)
- os.Remove(fdLock.Name())
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusDone}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
Jobs.Done()
}
// Was $1 touched?
- if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil {
+ if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil {
if inodePrev == nil {
- fd.Close()
runErr.Err = Err1WasTouched
errs <- runErr
return
}
- inode, err := inodeFromFile(fd)
- fd.Close()
if err != nil {
runErr.Err = err
errs <- runErr
}
if inodePrev != nil {
- if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil {
- inode, err := inodeFromFile(fd)
- fd.Close()
- if err == nil && !inode.Equals(inodePrev) {
+ if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil {
+ if !inode.Equals(inodePrev) {
runErr.Err = Err1WasTouched
errs <- runErr
return
// Does it produce both stdout and tmp?
fiStdout, err := os.Stat(fdStdout.Name())
if err != nil {
- runErr.Err = err
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
}
tmpExists = true
} else if !os.IsNotExist(err) {
- runErr.Err = err
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
if tmpExists {
fd, err = os.Open(tmpPath)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
defer fd.Close()
os.Remove(path.Join(cwdOrig, tgt))
err = ifcreate(fdDep, tgt)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
} else {
+ var hsh string
+ if hshPrev != "" {
+ _, err = fd.Seek(0, io.SeekStart)
+ if err != nil {
+ err = ErrLine(err)
+ goto Finish
+ }
+ hsh, err = fileHash(fd)
+ if err != nil {
+ err = ErrLine(err)
+ goto Finish
+ }
+ if hsh == hshPrev {
+ tracef(CDebug, "%s has same hash, not renaming", tgtOrig)
+ err = ErrLine(os.Remove(fd.Name()))
+ if err != nil {
+ goto Finish
+ }
+ err = ErrLine(os.Chtimes(path.Join(cwdOrig, tgt), finished, finished))
+ if err != nil {
+ goto Finish
+ }
+ if !NoSync {
+ err = ErrLine(syncDir(cwdOrig))
+ if err != nil {
+ goto Finish
+ }
+ }
+ err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hshPrev))
+ if err != nil {
+ goto Finish
+ }
+ goto RecCommit
+ }
+ }
if !NoSync {
- err = fd.Sync()
+ err = ErrLine(fd.Sync())
if err != nil {
goto Finish
}
}
- err = os.Rename(fd.Name(), path.Join(cwdOrig, tgt))
+ err = ErrLine(os.Rename(fd.Name(), path.Join(cwdOrig, tgt)))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(cwdOrig)
+ err = ErrLine(syncDir(cwdOrig))
if err != nil {
goto Finish
}
}
- err = depWrite(fdDep, cwdOrig, tgt)
+ err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh))
if err != nil {
goto Finish
}
}
+ RecCommit:
// Commit .rec
if !NoSync {
- err = fdDep.Sync()
+ err = ErrLine(fdDep.Sync())
if err != nil {
goto Finish
}
}
fdDepPath = path.Join(redoDir, tgt+DepSuffix)
- err = os.Rename(fdDep.Name(), fdDepPath)
+ err = ErrLine(os.Rename(fdDep.Name(), fdDepPath))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(redoDir)
+ err = ErrLine(syncDir(redoDir))
if err != nil {
goto Finish
}
depInfo, err := depRead(fdDepR)
fdDepR.Close()
if err != nil {
+ err = ErrLine(err)
goto Finish
}
ifchangeSeen := make(map[string]struct{}, len(depInfo.ifchanges))