From 6ef14f9de82e4a61532b84b149bde0c953d58a04 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Sat, 21 Nov 2020 22:55:08 +0300 Subject: [PATCH] redo-stamp with REDO_NO_HASH workability fix --- dep.go | 12 ++- ifchange.go | 4 +- main.go | 1 + ood.go | 240 ++++++++++++++++++++++++++++++---------------------- run.go | 20 +++-- 5 files changed, 166 insertions(+), 111 deletions(-) diff --git a/dep.go b/dep.go index 0f62d30..30a5cd6 100644 --- a/dep.go +++ b/dep.go @@ -68,12 +68,16 @@ func stamp(fdDep, src *os.File) error { if err != nil { return err } - trace(CDebug, "stamp: %s <- %s", fdDep.Name(), hsh) - return recfileWrite( - fdDep, + fields := []recfile.Field{ recfile.Field{Name: "Type", Value: DepTypeStamp}, recfile.Field{Name: "Hash", Value: hsh}, - ) + } + same := StampPrev == hsh + if same { + fields = append(fields, recfile.Field{Name: "Same", Value: "true"}) + } + trace(CDebug, "stamp: %s <- %s (%v)", fdDep.Name(), hsh, same) + return recfileWrite(fdDep, fields...) } func fileCtime(fd *os.File) (string, error) { diff --git a/ifchange.go b/ifchange.go index 73dcae2..3cd93fb 100644 --- a/ifchange.go +++ b/ifchange.go @@ -49,7 +49,7 @@ func ifchange(tgts []string) (bool, error) { if Force { ood = true } else { - ood, err = isOOD(Cwd, tgt, 0) + ood, err = isOOD(Cwd, tgt, 0, nil) if err != nil { return false, err } @@ -61,7 +61,7 @@ func ifchange(tgts []string) (bool, error) { trace(CDebug, "%s is source, not redoing", tgt) continue } - if err = runScript(tgt, errs); err != nil { + if err = runScript(tgt, errs, ""); err != nil { return false, err } if Force { diff --git a/main.go b/main.go index fdd0aac..ae49949 100644 --- a/main.go +++ b/main.go @@ -174,6 +174,7 @@ func main() { raw[0:4], raw[4:6], raw[6:8], raw[8:10], raw[10:], ) } + StampPrev = os.Getenv(EnvStampPrev) tgts := flag.Args() if len(tgts) == 0 { diff --git a/ood.go b/ood.go index 15d0ae5..9b6ebba 100644 --- a/ood.go +++ b/ood.go @@ -77,121 +77,126 @@ func isSrc(cwd, tgt string) bool { return true } -func isBuiltNow(fdDep *os.File) (bool, *recfile.Reader, error) { +type DepInfo struct { + build string + always bool + stamp string + stampSame bool + ifcreates []string + ifchanges []map[string]string +} + +func depRead(fdDep *os.File) (*DepInfo, error) { r := recfile.NewReader(fdDep) m, err := r.NextMap() if err != nil { - return false, nil, err + return nil, err + } + depInfo := DepInfo{} + if b := m["Build"]; b == "" { + return nil, errors.New(".dep missing Build:") + } else { + depInfo.build = b } - if m["Build"] == "" { - return false, r, errors.New(".dep missing Build:") + for { + m, err := r.NextMap() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + switch m["Type"] { + case DepTypeAlways: + depInfo.always = true + case DepTypeIfcreate: + dep := m["Target"] + if dep == "" { + return nil, errors.New("invalid format of .dep") + } + depInfo.ifcreates = append(depInfo.ifcreates, dep) + case DepTypeIfchange: + delete(m, "Type") + depInfo.ifchanges = append(depInfo.ifchanges, m) + case DepTypeStamp: + hsh := m["Hash"] + if hsh == "" { + return nil, errors.New("invalid format of .dep") + } + depInfo.stamp = hsh + depInfo.stampSame = m["Same"] == "true" + default: + return nil, errors.New("invalid format of .dep") + } } - return m["Build"] == BuildUUID, r, nil + return &depInfo, nil } -func rebuildStamped(cwd, tgt, depPath string) (string, error) { +func rebuildStamped(cwd, tgt, depPath, stampPrev string) (bool, error) { relTgt := cwdMustRel(cwd, tgt) errs := make(chan error, 1) - if err := runScript(relTgt, errs); err != nil { - return "", err + if err := runScript(relTgt, errs, stampPrev); err != nil { + return false, err } if err := <-errs; !isOkRun(err) { - return "", errors.New("build failed") + return false, errors.New("build failed") } fdDep, err := os.Open(depPath) if err != nil { - return "", err + return false, err } defer fdDep.Close() - builtNow, r, err := isBuiltNow(fdDep) + depInfo, err := depRead(fdDep) if err != nil { - return "", err - } - if !builtNow { - return "", errors.New("is not built") + return false, err } - var stampTheir string - for { - m, err := r.NextMap() - if err != nil { - if err == io.EOF { - break - } - return "", err - } - if m["Type"] == DepTypeStamp { - stampTheir = m["Hash"] - break - } + if depInfo.build != BuildUUID { + return false, errors.New("is not built") } - return stampTheir, nil + return depInfo.stampSame, nil +} + +func formDepPath(cwd, tgt string) string { + cwd, tgt = cwdAndTgt(path.Join(cwd, tgt)) + return path.Join(cwd, RedoDir, tgt+DepSuffix) } -func isOOD(cwd, tgtOrig string, level int) (bool, error) { +func isOOD(cwd, tgtOrig string, level int, depInfo *DepInfo) (bool, error) { indent := strings.Repeat(". ", level) trace(CDebug, "ood: %s%s checking", indent, tgtOrig) cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) - depPath := path.Join(cwd, RedoDir, tgt+DepSuffix) - fdDep, err := os.Open(depPath) - if err != nil { - trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) - return true, nil + depPath := formDepPath(cwd, tgt) + if depInfo == nil { + fdDep, err := os.Open(depPath) + if err != nil { + trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) + return true, nil + } + defer fdDep.Close() + depInfo, err = depRead(fdDep) + if err != nil { + return true, TgtErr{tgtOrig, err} + } } - defer fdDep.Close() - ood := false - builtNow, r, err := isBuiltNow(fdDep) - if err != nil { - return true, TgtErr{tgtOrig, err} - } - if builtNow { + if depInfo.build == BuildUUID { trace(CDebug, "ood: %s%s -> already built", indent, tgtOrig) return false, nil } - - var stampOur string - ifcreates := []map[string]string{} - ifchanges := []map[string]string{} - for { - m, err := r.NextMap() - if err != nil { - if err == io.EOF { - break - } - return true, TgtErr{tgtOrig, err} - } - switch m["Type"] { - case DepTypeAlways: - trace(CDebug, "ood: %s%s -> always", indent, tgtOrig) - ood = true - case DepTypeIfcreate: - ifcreates = append(ifcreates, m) - case DepTypeIfchange: - ifchanges = append(ifchanges, m) - case DepTypeStamp: - stampOur = m["Hash"] - trace(CDebug, "ood: %s%s -> stamped: %s", indent, tgtOrig, stampOur) - default: - return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} - } - } + ood := depInfo.always if ood { goto StampCheck } - for _, m := range ifcreates { - theirTgt := m["Target"] - if theirTgt == "" { - return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} - } - if _, err = os.Stat(path.Join(cwd, theirTgt)); err == nil { - trace(CDebug, "ood: %s%s -> created", indent, tgtOrig) + for _, dep := range depInfo.ifcreates { + if _, err := os.Stat(path.Join(cwd, dep)); err == nil { + trace(CDebug, "ood: %s%s -> %s created", indent, tgtOrig, dep) ood = true goto StampCheck } } - for _, m := range ifchanges { + for _, m := range depInfo.ifchanges { dep := m["Target"] theirTs := m["Ctime"] theirHsh := m["Hash"] @@ -199,6 +204,7 @@ func isOOD(cwd, tgtOrig string, level int) (bool, error) { return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} } trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) + fd, err := os.Open(path.Join(cwd, dep)) if err != nil { if os.IsNotExist(err) { @@ -209,29 +215,66 @@ func isOOD(cwd, tgtOrig string, level int) (bool, error) { return ood, TgtErr{tgtOrig, err} } defer fd.Close() - ts, err := fileCtime(fd) - if err != nil { - return ood, TgtErr{tgtOrig, err} + + var depDepInfo *DepInfo + if !(dep == tgt || isSrc(cwd, dep)) { + trace(CDebug, "ood: %s%s -> %s: prereading .dep", indent, tgtOrig, dep) + depFdDep, err := os.Open(formDepPath(cwd, dep)) + if err != nil { + return ood, TgtErr{path.Join(tgtOrig, dep), err} + } + defer depFdDep.Close() + depDepInfo, err = depRead(depFdDep) + if err != nil { + return ood, TgtErr{path.Join(tgtOrig, dep), err} + } } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) - } else if NoHash || theirHsh == "" { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) - ood = true - goto StampCheck - } else { - hsh, err := fileHash(fd) + + if depDepInfo != nil && depDepInfo.build == BuildUUID { + trace( + CDebug, "ood: %s%s -> %s: .dep says build is same", + indent, tgtOrig, dep, + ) + if !depDepInfo.stampSame { + trace( + CDebug, "ood: %s%s -> %s: .dep says stamp is not same", + indent, tgtOrig, dep, + ) + ood = true + return ood, nil + } + trace( + CDebug, "ood: %s%s -> %s: .dep says stamp is same", + indent, tgtOrig, dep, + ) + continue + } + + if depDepInfo == nil || !depDepInfo.always && depDepInfo.stamp == "" { + ts, err := fileCtime(fd) if err != nil { return ood, TgtErr{tgtOrig, err} } - if theirHsh != hsh { - trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) + if theirTs == ts { + trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) + } else if NoHash || theirHsh == "" { + trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) ood = true goto StampCheck + } else { + hsh, err := fileHash(fd) + if err != nil { + return ood, TgtErr{tgtOrig, err} + } + if theirHsh != hsh { + trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) + ood = true + goto StampCheck + } + trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) } - trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) } - fd.Close() + if dep == tgt { trace(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep) continue @@ -240,7 +283,8 @@ func isOOD(cwd, tgtOrig string, level int) (bool, error) { trace(CDebug, "ood: %s%s -> %s: is source", indent, tgtOrig, dep) continue } - depOod, err := isOOD(cwd, dep, level+1) + + depOod, err := isOOD(cwd, dep, level+1, depDepInfo) if err != nil { return ood, TgtErr{tgtOrig, err} } @@ -253,16 +297,14 @@ func isOOD(cwd, tgtOrig string, level int) (bool, error) { } StampCheck: - if ood && stampOur != "" { + if ood && depInfo.stamp != "" { trace(CDebug, "ood: %s%s run, because stamped", indent, tgtOrig) - stampTheir, err := rebuildStamped(cwd, tgt, depPath) + stampSame, err := rebuildStamped(cwd, tgt, depPath, depInfo.stamp) if err != nil { return true, TgtErr{tgtOrig, err} } - trace(CDebug, "ood: %s%s -> stamp: %s %s", indent, tgtOrig, stampOur, stampTheir) - if stampOur == stampTheir { - ood = false - } + trace(CDebug, "ood: %s%s -> stamp: same: %v", indent, tgtOrig, stampSame) + ood = !stampSame } trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) return ood, nil diff --git a/run.go b/run.go index c73d8b4..eb67304 100644 --- a/run.go +++ b/run.go @@ -47,6 +47,7 @@ const ( EnvStderrKeep = "REDO_LOGS" EnvStderrSilent = "REDO_SILENT" EnvNoSync = "REDO_NO_SYNC" + EnvStampPrev = "REDO_STAMP_PREV" RedoDir = ".redo" LockSuffix = ".lock" @@ -61,6 +62,7 @@ var ( StderrKeep bool = false StderrSilent bool = false StderrPrefix string + StampPrev string flagTrace = flag.Bool("x", false, "trace current target (sh -x) (set REDO_TRACE=1 for others too)") flagStderrKeep = flag.Bool("logs", false, "keep job's stderr (REDO_LOGS=1)") @@ -162,7 +164,7 @@ func syncDir(dir string) error { return err } -func runScript(tgtOrig string, errs chan error) error { +func runScript(tgtOrig string, errs chan error, stampPrev string) error { cwd, tgt := cwdAndTgt(tgtOrig) redoDir := path.Join(cwd, RedoDir) if err := mkdirs(redoDir); err != nil { @@ -199,20 +201,21 @@ func runScript(tgtOrig string, errs chan error) error { unix.Flock(int(fdLock.Fd()), unix.LOCK_EX) lockRelease() trace(CDebug, "waiting done: %s", tgtOrig) - var builtNow bool + var depInfo *DepInfo fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - err = errors.New("was not built") + err = errors.New("was not built: no .dep") } goto Finish } - builtNow, _, err = isBuiltNow(fdDep) + defer fdDep.Close() + depInfo, err = depRead(fdDep) if err != nil { goto Finish } - if !builtNow { - err = errors.New("was not built") + if depInfo.build != BuildUUID { + err = errors.New("was not built: build differs") } Finish: if err != nil { @@ -343,6 +346,11 @@ func runScript(tgtOrig string, errs chan error) error { cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID)) + + if stampPrev != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvStampPrev, stampPrev)) + } + childStderrPrefix := tempsuffix() cmd.Env = append(cmd.Env, fmt.Sprintf( "%s=%s", EnvStderrPrefix, childStderrPrefix, -- 2.44.0