From: Sergey Matveev Date: Sun, 22 Nov 2020 11:45:56 +0000 (+0300) Subject: Removed hashless mode, small bugfixes, tai64nlocal X-Git-Tag: v0.2.0~10 X-Git-Url: http://www.git.cypherpunks.ru/?p=goredo.git;a=commitdiff_plain;h=f8a8a335216a6ec9aa6119d473d2f6aeb43958ca Removed hashless mode, small bugfixes, tai64nlocal --- diff --git a/.gitignore b/.gitignore index 40fa2fa..99bbf7d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /redo-log /redo-stamp /redo-whichdo +/tai64nlocal diff --git a/README b/README index 022ae20..edb5b76 100644 --- a/README +++ b/README @@ -37,35 +37,34 @@ NOTES *goredo-notes* FEATURES *goredo-features* -* explicit check that stdout and $3 are not written simultaneously -* explicit check (similar to apenwarr/redo's one) that generated target - was not modified "externally" outside the redo, preventing its - overwriting, but continuing the build +* explicit useful and convenient checks from apenwarr/redo: + * check that $1 was not touched during .do execution + * check that stdout and $3 are not written simultaneously + * check that generated target was not modified "externally" outside + the redo, preventing its overwriting, but continuing the build * targets, dependency information and their directories are explicitly synced (can be disabled, should work faster) -* file's change is detected by comparing its ctime and, if it differs, - its BLAKE2b hash. Hash checking is done to prevent false positives - (can be disabled, will work faster) +* file's change is detected by comparing its ctime and BLAKE2b hash * files creation is umask-friendly (unlike mkstemp() used in redo-c) +* parallel build with jobs limit, optionally in infinite mode * coloured messages (can be disabled) * verbose debug messages, including out-of-date determination, PIDs, - lock acquirings/releases -* parallel build with jobs limit, optionally in infinite mode -* optional display of each target's execution time + lock and jobserver acquirings/releases +* displaying of each target's execution time * each target's stderr can be displayed with the PID * target's stderr can be stored on the disk with TAI64N timestamp - prefixes for each line (you can use tai64nlocal utility from - daemontools (http://cr.yp.to/daemontools/tai64nlocal.html) to convert - them to local time). Captures can be viewed any time later - + prefixes for each line. To convert them to localtime you can use either + tai64nlocal utility from daemontools (http://cr.yp.to/daemontools.html) + or make a symlink, to use built-in slower decoder: > + $ ln -s goredo tai64nlocal +< COMMANDS *goredo-commands* * redo-ifchange, redo-ifcreate, redo-always (useful with redo-stamp) * redo -- same as redo-ifchange, but forcefully and *sequentially* run specified targets * redo-log -- display TAI64N timestamped last stderr of the target -* redo-stamp -- record stamp dependency and consider it non out-of-date - if stamp equals to the previous one +* redo-stamp -- record stamp dependency. Nothing more, just dummy * redo-cleanup -- removes either temporary, or log files, or everything related to goredo * redo-whichdo -- .do search paths for specified target (similar to diff --git a/dep.go b/dep.go index 30a5cd6..e6663e6 100644 --- a/dep.go +++ b/dep.go @@ -22,6 +22,7 @@ package main import ( "bufio" "encoding/hex" + "errors" "fmt" "io" "os" @@ -33,10 +34,6 @@ import ( "golang.org/x/crypto/blake2b" ) -const EnvNoHash = "REDO_NO_HASH" - -var NoHash bool - func recfileWrite(fdDep *os.File, fields ...recfile.Field) error { w := recfile.NewWriter(fdDep) if _, err := w.RecordStart(); err != nil { @@ -68,16 +65,12 @@ func stamp(fdDep, src *os.File) error { if err != nil { return err } - fields := []recfile.Field{ + trace(CDebug, "stamp: %s <- %s", fdDep.Name(), hsh) + return recfileWrite( + fdDep, recfile.Field{Name: "Type", Value: DepTypeStamp}, recfile.Field{Name: "Hash", Value: hsh}, - } - same := StampPrev == hsh - if same { - fields = append(fields, recfile.Field{Name: "Same", Value: "true"}) - } - trace(CDebug, "stamp: %s <- %s (%v)", fdDep.Name(), hsh, same) - return recfileWrite(fdDep, fields...) + ) } func fileCtime(fd *os.File) (string, error) { @@ -112,20 +105,17 @@ func writeDep(fdDep *os.File, cwd, tgt string) error { if err != nil { return err } - fields := []recfile.Field{ + hsh, err := fileHash(fd) + if err != nil { + return err + } + return recfileWrite( + fdDep, recfile.Field{Name: "Type", Value: DepTypeIfchange}, recfile.Field{Name: "Target", Value: tgt}, recfile.Field{Name: "Ctime", Value: ts}, - } - var hsh string - if !NoHash { - hsh, err = fileHash(fd) - if err != nil { - return err - } - fields = append(fields, recfile.Field{Name: "Hash", Value: hsh}) - } - return recfileWrite(fdDep, fields...) + recfile.Field{Name: "Hash", Value: hsh}, + ) } func writeDeps(fdDep *os.File, tgts []string) error { @@ -148,3 +138,56 @@ func writeDeps(fdDep *os.File, tgts []string) error { } return nil } + +type DepInfo struct { + build string + always bool + stamp string + ifcreates []string + ifchanges []map[string]string +} + +func depRead(fdDep *os.File) (*DepInfo, error) { + r := recfile.NewReader(fdDep) + m, err := r.NextMap() + if err != nil { + return nil, err + } + depInfo := DepInfo{} + if b := m["Build"]; b == "" { + return nil, errors.New(".dep missing Build:") + } else { + depInfo.build = b + } + for { + m, err := r.NextMap() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + switch m["Type"] { + case DepTypeAlways: + depInfo.always = true + case DepTypeIfcreate: + dep := m["Target"] + if dep == "" { + return nil, errors.New("invalid format of .dep") + } + depInfo.ifcreates = append(depInfo.ifcreates, dep) + case DepTypeIfchange: + delete(m, "Type") + depInfo.ifchanges = append(depInfo.ifchanges, m) + case DepTypeStamp: + hsh := m["Hash"] + if hsh == "" { + return nil, errors.New("invalid format of .dep") + } + depInfo.stamp = hsh + default: + return nil, errors.New("invalid format of .dep") + } + } + return &depInfo, nil +} diff --git a/ifchange.go b/ifchange.go index 3cd93fb..9e39e7f 100644 --- a/ifchange.go +++ b/ifchange.go @@ -17,39 +17,176 @@ along with this program. If not, see . package main -import "sync" - -var ( - Force bool = false - Jobs sync.WaitGroup +import ( + "os" + "path" + "strings" ) -func isOkRun(err error) bool { - if err == nil { - return true +func collectDeps( + cwd, tgtOrig string, + level int, + deps map[string]map[string]struct{}, +) []string { + cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) + depPath := path.Join(cwd, RedoDir, tgt+DepSuffix) + fdDep, err := os.Open(depPath) + if err != nil { + return nil + } + depInfo, err := depRead(fdDep) + fdDep.Close() + if err != nil { + return nil } - if err, ok := err.(RunErr); ok && err.Err == nil { - trace(CRedo, "%s", err.Name()) - return true + var alwayses []string + returnReady := false + tgtRel := cwdMustRel(cwd, tgt) + if depInfo.always { + if depInfo.build == BuildUUID { + trace( + CDebug, "ood: %s%s always, but already build", + strings.Repeat(". ", level), tgtOrig, + ) + returnReady = true + } else { + trace(CDebug, "ood: %s%s always", strings.Repeat(". ", level), tgtOrig) + alwayses = append(alwayses, tgtRel) + returnReady = true + } } - trace(CErr, "%s", err) - return false + for _, m := range depInfo.ifchanges { + dep := m["Target"] + if dep == "" { + return alwayses + } + if dep == tgt || isSrc(cwd, dep) { + continue + } + if !returnReady { + depRel := cwdMustRel(cwd, dep) + if m, ok := deps[depRel]; ok { + m[tgtRel] = struct{}{} + } else { + m = make(map[string]struct{}, 0) + m[tgtRel] = struct{}{} + deps[depRel] = m + } + alwayses = append(alwayses, collectDeps(cwd, dep, level+1, deps)...) + } + } + return alwayses } -func ifchange(tgts []string) (bool, error) { +func buildDependants(tgts []string) map[string]struct{} { + defer Jobs.Wait() + trace(CDebug, "collecting deps") + seen := map[string]struct{}{} + deps := map[string]map[string]struct{}{} + for _, tgt := range tgts { + for _, tgt := range collectDeps(Cwd, tgt, 0, deps) { + seen[tgt] = struct{}{} + } + } + if len(seen) == 0 { + return nil + } + + trace(CDebug, "building %d alwayses: %v", len(seen), seen) + errs := make(chan error, len(seen)) + for tgt, _ := range seen { + if err := runScript(tgt, errs); err != nil { + trace(CErr, "always run error: %s, skipping dependants", err) + return nil + } + } + ok := true + for i := 0; i < len(seen); i++ { + ok = ok && isOkRun(<-errs) + } + Jobs.Wait() + close(errs) + if !ok { + trace(CDebug, "alwayses failed, skipping depdendants") + return nil + } + + queueSrc := make([]string, 0, len(seen)) + for tgt, _ := range seen { + queueSrc = append(queueSrc, tgt) + } + if len(queueSrc) == 0 { + return seen + } + levelOrig := Level + defer func() { + Level = levelOrig + }() + Level = 1 + +RebuildDeps: + trace(CDebug, "checking %d dependant targets: %v", len(queueSrc), queueSrc) + queue := []string{} + for _, tgt := range queueSrc { + for dep, _ := range deps[tgt] { + queue = append(queue, dep) + } + } + trace(CDebug, "building %d dependant targets: %v", len(queue), queue) + errs = make(chan error, len(queue)) + jobs := 0 + queueSrc = []string{} + for _, tgt := range queue { + ood, err := isOOD(Cwd, tgt, 0, seen) + if err != nil { + trace(CErr, "dependant error: %s, skipping dependants", err) + return nil + } + if !ood { + continue + } + if err := runScript(tgt, errs); err != nil { + trace(CErr, "dependant error: %s, skipping dependants", err) + return nil + } + queueSrc = append(queueSrc, tgt) + seen[tgt] = struct{}{} + jobs++ + } + for i := 0; i < jobs; i++ { + ok = ok && isOkRun(<-errs) + } + if !ok { + trace(CDebug, "dependants failed, skipping them") + return nil + } + Jobs.Wait() + close(errs) + if jobs == 0 { + return seen + } + Level++ + goto RebuildDeps +} + +func ifchange(tgts []string, forced bool) (bool, error) { jsInit() defer jsAcquire("ifchange exiting") defer Jobs.Wait() - errs := make(chan error, len(tgts)) + seen := buildDependants(tgts) + trace(CDebug, "building %d targets: %v", len(tgts), tgts) jobs := 0 - ok := true + errs := make(chan error, len(tgts)) + var ood bool var err error for _, tgt := range tgts { - var ood bool - if Force { - ood = true - } else { - ood, err = isOOD(Cwd, tgt, 0, nil) + if _, ok := seen[tgt]; ok { + trace(CDebug, "%s was already build as a dependenant", tgt) + continue + } + ood = true + if !forced { + ood, err = isOOD(Cwd, tgt, 0, seen) if err != nil { return false, err } @@ -61,23 +198,14 @@ func ifchange(tgts []string) (bool, error) { trace(CDebug, "%s is source, not redoing", tgt) continue } - if err = runScript(tgt, errs, ""); err != nil { + if err = runScript(tgt, errs); err != nil { return false, err } - if Force { - // Sequentially run jobs - err = <-errs - Jobs.Wait() - if isOkRun(err) { - continue - } - return false, nil - } jobs++ } - for i := 0; i < jobs; i++ { - err = <-errs - ok = ok && isOkRun(err) + ok := true + for ; jobs > 0; jobs-- { + ok = ok && isOkRun(<-errs) } return ok, nil } diff --git a/main.go b/main.go index ae49949..c01ab80 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ along with this program. If not, see . package main import ( + "bufio" "crypto/rand" "flag" "fmt" @@ -84,11 +85,13 @@ func main() { "redo-stamp", "redo-whichdo", } { + fmt.Println(os.Args[0], "<-", cmdName) if err := os.Symlink(os.Args[0], cmdName); err != nil { rc = 1 log.Println(err) } } + fmt.Println("no creating optional:", os.Args[0], "<- tai64nlocal") os.Exit(rc) } log.SetFlags(0) @@ -100,7 +103,6 @@ func main() { } NoColor = os.Getenv(EnvNoColor) != "" - NoHash = os.Getenv(EnvNoHash) == "1" NoSync = os.Getenv(EnvNoSync) == "1" TopDir = os.Getenv(EnvTopDir) @@ -174,7 +176,6 @@ func main() { raw[0:4], raw[4:6], raw[6:8], raw[8:10], raw[10:], ) } - StampPrev = os.Getenv(EnvStampPrev) tgts := flag.Args() if len(tgts) == 0 { @@ -188,10 +189,14 @@ func main() { CmdSwitch: switch cmdName { case "redo": - Force = true - ok, err = ifchange(tgts) + for _, tgt := range tgts { + ok, err = ifchange([]string{tgt}, true) + if err != nil || !ok { + break + } + } case "redo-ifchange": - ok, err = ifchange(tgts) + ok, err = ifchange(tgts, false) writeDeps(fdDep, tgts) case "redo-ifcreate": if fdDep == nil { @@ -262,6 +267,10 @@ CmdSwitch: fmt.Println(cwdMustRel(cwd, m["Target"])) } ok = doFile != "" + case "tai64nlocal": + bw := bufio.NewWriter(os.Stdout) + err = tai64nLocal(bw, os.Stdin) + bw.Flush() default: log.Fatalln("unknown command", cmdName) } diff --git a/ood.go b/ood.go index 9b6ebba..95c3ed2 100644 --- a/ood.go +++ b/ood.go @@ -22,13 +22,10 @@ package main import ( "errors" "fmt" - "io" "os" "path" "path/filepath" "strings" - - "go.cypherpunks.ru/recfile" ) const ( @@ -77,122 +74,33 @@ func isSrc(cwd, tgt string) bool { return true } -type DepInfo struct { - build string - always bool - stamp string - stampSame bool - ifcreates []string - ifchanges []map[string]string -} - -func depRead(fdDep *os.File) (*DepInfo, error) { - r := recfile.NewReader(fdDep) - m, err := r.NextMap() - if err != nil { - return nil, err - } - depInfo := DepInfo{} - if b := m["Build"]; b == "" { - return nil, errors.New(".dep missing Build:") - } else { - depInfo.build = b - } - for { - m, err := r.NextMap() - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - switch m["Type"] { - case DepTypeAlways: - depInfo.always = true - case DepTypeIfcreate: - dep := m["Target"] - if dep == "" { - return nil, errors.New("invalid format of .dep") - } - depInfo.ifcreates = append(depInfo.ifcreates, dep) - case DepTypeIfchange: - delete(m, "Type") - depInfo.ifchanges = append(depInfo.ifchanges, m) - case DepTypeStamp: - hsh := m["Hash"] - if hsh == "" { - return nil, errors.New("invalid format of .dep") - } - depInfo.stamp = hsh - depInfo.stampSame = m["Same"] == "true" - default: - return nil, errors.New("invalid format of .dep") - } - } - return &depInfo, nil -} - -func rebuildStamped(cwd, tgt, depPath, stampPrev string) (bool, error) { - relTgt := cwdMustRel(cwd, tgt) - errs := make(chan error, 1) - if err := runScript(relTgt, errs, stampPrev); err != nil { - return false, err - } - if err := <-errs; !isOkRun(err) { - return false, errors.New("build failed") - } +func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, error) { + indent := strings.Repeat(". ", level) + trace(CDebug, "ood: %s%s checking", indent, tgtOrig) + cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) + depPath := path.Join(cwd, RedoDir, tgt+DepSuffix) fdDep, err := os.Open(depPath) if err != nil { - return false, err + trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) + return true, nil } - defer fdDep.Close() depInfo, err := depRead(fdDep) + fdDep.Close() if err != nil { - return false, err - } - if depInfo.build != BuildUUID { - return false, errors.New("is not built") - } - return depInfo.stampSame, nil -} - -func formDepPath(cwd, tgt string) string { - cwd, tgt = cwdAndTgt(path.Join(cwd, tgt)) - return path.Join(cwd, RedoDir, tgt+DepSuffix) -} - -func isOOD(cwd, tgtOrig string, level int, depInfo *DepInfo) (bool, error) { - indent := strings.Repeat(". ", level) - trace(CDebug, "ood: %s%s checking", indent, tgtOrig) - cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) - depPath := formDepPath(cwd, tgt) - if depInfo == nil { - fdDep, err := os.Open(depPath) - if err != nil { - trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) - return true, nil - } - defer fdDep.Close() - depInfo, err = depRead(fdDep) - if err != nil { - return true, TgtErr{tgtOrig, err} - } + return true, TgtErr{tgtOrig, err} } if depInfo.build == BuildUUID { trace(CDebug, "ood: %s%s -> already built", indent, tgtOrig) return false, nil } - ood := depInfo.always - if ood { - goto StampCheck - } + ood := false for _, dep := range depInfo.ifcreates { if _, err := os.Stat(path.Join(cwd, dep)); err == nil { trace(CDebug, "ood: %s%s -> %s created", indent, tgtOrig, dep) ood = true - goto StampCheck + goto Done } } @@ -210,70 +118,32 @@ func isOOD(cwd, tgtOrig string, level int, depInfo *DepInfo) (bool, error) { if os.IsNotExist(err) { trace(CDebug, "ood: %s%s -> %s: not exists", indent, tgtOrig, dep) ood = true - goto StampCheck + goto Done } return ood, TgtErr{tgtOrig, err} } defer fd.Close() - var depDepInfo *DepInfo - if !(dep == tgt || isSrc(cwd, dep)) { - trace(CDebug, "ood: %s%s -> %s: prereading .dep", indent, tgtOrig, dep) - depFdDep, err := os.Open(formDepPath(cwd, dep)) - if err != nil { - return ood, TgtErr{path.Join(tgtOrig, dep), err} - } - defer depFdDep.Close() - depDepInfo, err = depRead(depFdDep) - if err != nil { - return ood, TgtErr{path.Join(tgtOrig, dep), err} - } - } - - if depDepInfo != nil && depDepInfo.build == BuildUUID { - trace( - CDebug, "ood: %s%s -> %s: .dep says build is same", - indent, tgtOrig, dep, - ) - if !depDepInfo.stampSame { - trace( - CDebug, "ood: %s%s -> %s: .dep says stamp is not same", - indent, tgtOrig, dep, - ) - ood = true - return ood, nil - } - trace( - CDebug, "ood: %s%s -> %s: .dep says stamp is same", - indent, tgtOrig, dep, - ) - continue + ts, err := fileCtime(fd) + if err != nil { + return ood, TgtErr{tgtOrig, err} } - - if depDepInfo == nil || !depDepInfo.always && depDepInfo.stamp == "" { - ts, err := fileCtime(fd) + if theirTs == ts { + trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) + } else { + trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) + hsh, err := fileHash(fd) if err != nil { return ood, TgtErr{tgtOrig, err} } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) - } else if NoHash || theirHsh == "" { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) + if theirHsh != hsh { + trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) ood = true - goto StampCheck - } else { - hsh, err := fileHash(fd) - if err != nil { - return ood, TgtErr{tgtOrig, err} - } - if theirHsh != hsh { - trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) - ood = true - goto StampCheck - } - trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) + goto Done } + trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) } + fd.Close() // optimization not to hold it for long if dep == tgt { trace(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep) @@ -284,28 +154,24 @@ func isOOD(cwd, tgtOrig string, level int, depInfo *DepInfo) (bool, error) { continue } - depOod, err := isOOD(cwd, dep, level+1, depDepInfo) + if _, ok := seen[cwdMustRel(cwd, dep)]; ok { + trace(CDebug, "ood: %s%s -> %s: was always built", indent, tgtOrig, dep) + continue + } + + depOod, err := isOOD(cwd, dep, level+1, seen) if err != nil { return ood, TgtErr{tgtOrig, err} } if depOod { trace(CDebug, "ood: %s%s -> %s: ood", indent, tgtOrig, dep) ood = true - goto StampCheck + goto Done } trace(CDebug, "ood: %s%s -> %s: !ood", indent, tgtOrig, dep) } -StampCheck: - if ood && depInfo.stamp != "" { - trace(CDebug, "ood: %s%s run, because stamped", indent, tgtOrig) - stampSame, err := rebuildStamped(cwd, tgt, depPath, depInfo.stamp) - if err != nil { - return true, TgtErr{tgtOrig, err} - } - trace(CDebug, "ood: %s%s -> stamp: same: %v", indent, tgtOrig, stampSame) - ood = !stampSame - } +Done: trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) return ood, nil } diff --git a/run.go b/run.go index eb67304..0f7277b 100644 --- a/run.go +++ b/run.go @@ -31,6 +31,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -47,7 +48,6 @@ const ( EnvStderrKeep = "REDO_LOGS" EnvStderrSilent = "REDO_SILENT" EnvNoSync = "REDO_NO_SYNC" - EnvStampPrev = "REDO_STAMP_PREV" RedoDir = ".redo" LockSuffix = ".lock" @@ -62,7 +62,7 @@ var ( StderrKeep bool = false StderrSilent bool = false StderrPrefix string - StampPrev string + Jobs sync.WaitGroup flagTrace = flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)") flagStderrKeep = flag.Bool("logs", false, "keep job's stderr (REDO_LOGS=1)") @@ -113,23 +113,24 @@ func tempfile(dir, prefix string) (*os.File, error) { return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666)) } -func isModified(cwd, redoDir, tgt string) (bool, error) { +func isModified(cwd, redoDir, tgt string) (bool, string, error) { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - return false, nil + return false, "", nil } - return false, err + return false, "", err } defer fdDep.Close() r := recfile.NewReader(fdDep) + var ourTs string for { m, err := r.NextMap() if err != nil { if err == io.EOF { break } - return false, err + return false, "", err } if m["Target"] != tgt { continue @@ -137,21 +138,21 @@ func isModified(cwd, redoDir, tgt string) (bool, error) { fd, err := os.Open(path.Join(cwd, tgt)) if err != nil { if os.IsNotExist(err) { - return false, nil + return false, "", nil } - return false, err + return false, "", err } defer fd.Close() - ourTs, err := fileCtime(fd) + ourTs, err = fileCtime(fd) if err != nil { - return false, err + return false, "", err } if ourTs != m["Ctime"] { - return true, nil + return true, ourTs, nil } break } - return false, nil + return false, ourTs, nil } func syncDir(dir string) error { @@ -164,7 +165,7 @@ func syncDir(dir string) error { return err } -func runScript(tgtOrig string, errs chan error, stampPrev string) error { +func runScript(tgtOrig string, errs chan error) error { cwd, tgt := cwdAndTgt(tgtOrig) redoDir := path.Join(cwd, RedoDir) if err := mkdirs(redoDir); err != nil { @@ -227,7 +228,7 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { } // Check if target is not modified externally - modified, err := isModified(cwd, redoDir, tgt) + modified, tsPrev, err := isModified(cwd, redoDir, tgt) if err != nil { lockRelease() return TgtErr{tgtOrig, err} @@ -235,8 +236,10 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { if modified { trace(CWarn, "%s externally modified: not redoing", tgtOrig) lockRelease() - errs <- nil - return TgtErr{tgtOrig, err} + go func() { + errs <- nil + }() + return nil } // Start preparing .dep @@ -347,10 +350,6 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID)) - if stampPrev != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvStampPrev, stampPrev)) - } - childStderrPrefix := tempsuffix() cmd.Env = append(cmd.Env, fmt.Sprintf( "%s=%s", EnvStderrPrefix, childStderrPrefix, @@ -366,7 +365,9 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { } else { cmd.ExtraFiles = append(cmd.ExtraFiles, JSR) cmd.ExtraFiles = append(cmd.ExtraFiles, JSW) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1)) + cmd.Env = append(cmd.Env, fmt.Sprintf( + "%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1, + )) fdNum += 2 } @@ -458,6 +459,18 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { return } + // Was $1 touched? + if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { + ts, err := fileCtime(fd) + fd.Close() + if err == nil && ts != tsPrev { + runErr.Err = errors.New("$1 was explicitly touched") + errs <- runErr + fd.Close() + return + } + } + // Does it produce both stdout and tmp? fiStdout, err := os.Stat(fdStdout.Name()) if err != nil { @@ -544,3 +557,15 @@ func runScript(tgtOrig string, errs chan error, stampPrev string) error { }() return nil } + +func isOkRun(err error) bool { + if err == nil { + return true + } + if err, ok := err.(RunErr); ok && err.Err == nil { + trace(CRedo, "%s", err.Name()) + return true + } + trace(CErr, "%s", err) + return false +} diff --git a/tai64n.go b/tai64n.go index f2b4791..e6dd28e 100644 --- a/tai64n.go +++ b/tai64n.go @@ -1,14 +1,82 @@ +/* +goredo -- redo implementation on pure Go +Copyright (C) 2020 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + package main import ( + "bufio" "encoding/binary" + "encoding/hex" + "errors" + "io" + "strings" "time" ) -type TAI64N [12]byte +const ( + TAI64NSize = 12 + TAI64NBase = 0x400000000000000a + TAI64NLocalFmt = "2006-01-02 15:04:05.000000000" +) + +type TAI64N [TAI64NSize]byte func tai64nNow(ts *TAI64N) { t := time.Now() - binary.BigEndian.PutUint64(ts[:], uint64(0x400000000000000a)+uint64(t.Unix())) + binary.BigEndian.PutUint64(ts[:], uint64(TAI64NBase)+uint64(t.Unix())) binary.BigEndian.PutUint32(ts[8:], uint32(t.Nanosecond())) } + +func tai64nLocal(dst io.StringWriter, src io.Reader) error { + scanner := bufio.NewScanner(src) + var err error + var s string + var sep int + var ts []byte + var secs int64 + var nano int64 + var t time.Time + for { + if !scanner.Scan() { + if err = scanner.Err(); err != nil { + return err + } + break + } + s = scanner.Text() + + if s[0] != '@' { + dst.WriteString(s + "\n") + } + sep = strings.IndexByte(s, byte(' ')) + if sep == -1 { + dst.WriteString(s + "\n") + } + ts, err = hex.DecodeString(s[1:sep]) + if err != nil { + return err + } + if len(ts) != TAI64NSize { + return errors.New("invalid ts length") + } + secs = int64(binary.BigEndian.Uint64(ts[:8])) + nano = int64(binary.BigEndian.Uint32(ts[8:])) + t = time.Unix(secs-TAI64NBase, nano) + dst.WriteString(t.Format(TAI64NLocalFmt) + s[sep:] + "\n") + } + return nil +} diff --git a/usage.go b/usage.go index d29e1ca..171e0d3 100644 --- a/usage.go +++ b/usage.go @@ -26,7 +26,7 @@ import ( ) const ( - Version = "0.1.0" + Version = "0.2.0" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License. @@ -53,8 +53,8 @@ License GPLv3: GNU GPL version 3 This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. -redo, redo-{always,cleanup,dot,ifchange,ifcreate,log,stamp,whichdo} must be -linked to goredo executable. It determines the command by its own name. +redo, redo-{always,cleanup,dot,ifchange,ifcreate,log,stamp,whichdo} must +be linked to goredo executable. It determines the command by its own name. You can create them by running: goredo -symlinks. * redo [options] [target ...] @@ -76,7 +76,9 @@ You can create them by running: goredo -symlinks. last build is kept. You must enable stderr keeping with either -logs, or REDO_LOGS=1 * redo-stamp < [$3] - record stamp dependency for current target. Unusable outside .do + record stamp dependency for current target. Unusable outside .do. + Stamp dependency does not play any role, as all targets are hashed + anyway * redo-whichdo target display .do search paths for specified target. Exits successfully if the last .do in output if the found existing one @@ -87,7 +89,6 @@ Options: fmt.Fprintln(os.Stderr, ` Additional environment variables: NO_COLOR -- disable messages colouring - REDO_NO_HASH -- disable dependencies (except redo-stamp-ed) hashing REDO_NO_SYNC -- disable files/directories explicit filesystem syncing REDO_TOP_DIR -- do not search for .do above that directory (it can contain .redo/top as an alternative)`)