From 4dea8061673b04d0225887f1f8d73392823e4e9e Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Wed, 26 Jan 2022 16:17:15 +0300 Subject: [PATCH] redo-depfix --- .gitignore | 1 + depfix.go | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/cmds.texi | 11 +++ doc/news.texi | 6 ++ doc/ood.texi | 6 ++ main.go | 4 + ood.go | 4 +- usage.go | 9 ++- 8 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 depfix.go diff --git a/.gitignore b/.gitignore index 60a3ad6..fdf24f9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /redo-affects /redo-always /redo-cleanup +/redo-depfix /redo-dot /redo-ifchange /redo-ifcreate diff --git a/depfix.go b/depfix.go new file mode 100644 index 0000000..0e1d54c --- /dev/null +++ b/depfix.go @@ -0,0 +1,214 @@ +/* +goredo -- djb's redo implementation on pure Go +Copyright (C) 2020-2022 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "errors" + "io" + "os" + "path" + "strings" + + "go.cypherpunks.ru/recfile" +) + +func depFix(root string) error { + tracef(CDebug, "depfix: entering %s", root) + dir, err := os.Open(root) + if err != nil { + return err + } + defer dir.Close() + for { + fis, err := dir.Readdir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + return err + } + for _, fi := range fis { + if fi.IsDir() { + if err = depFix(path.Join(root, fi.Name())); err != nil { + return err + } + } + } + } + dir.Close() + + redoDir := path.Join(root, RedoDir) + dir, err = os.Open(redoDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer dir.Close() + redoDirChanged := false + for { + fis, err := dir.Readdir(1 << 10) + if err != nil { + if err == io.EOF { + break + } + return err + } + for _, fi := range fis { + if !strings.HasSuffix(fi.Name(), DepSuffix) { + continue + } + tracef(CDebug, "depfix: checking %s/%s", root, fi.Name()) + fdDepPath := path.Join(redoDir, fi.Name()) + fdDep, err := os.Open(fdDepPath) + if err != nil { + return err + } + defer fdDep.Close() + r := recfile.NewReader(fdDep) + var fieldses [][]recfile.Field + depChanged := false + for { + fields, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } + fieldses = append(fieldses, fields) + m := make(map[string]string, len(fields)) + for _, f := range fields { + m[f.Name] = f.Value + } + if m["Type"] != DepTypeIfchange { + continue + } + dep := m["Target"] + if dep == "" { + return ErrMissingTarget + } + tracef(CDebug, "depfix: checking %s/%s -> %s", root, fi.Name(), dep) + theirInode, err := inodeFromRec(m) + if err != nil { + return err + } + theirHsh := m["Hash"] + fd, err := os.Open(path.Join(root, dep)) + if err != nil { + if os.IsNotExist(err) { + tracef( + CDebug, "depfix: %s/%s -> %s: not exists", + root, fi.Name(), dep, + ) + continue + } + return err + } + inode, err := inodeFromFile(fd) + if err != nil { + return err + } + if inode.Size != theirInode.Size { + tracef( + CDebug, "depfix: %s/%s -> %s: size differs", + root, fi.Name(), dep, + ) + fd.Close() + continue + } + if inode.Equals(theirInode) { + tracef( + CDebug, "depfix: %s/%s -> %s: inode is equal", + root, fi.Name(), dep, + ) + fd.Close() + continue + } + hsh, err := fileHash(fd) + fd.Close() + if err != nil { + return err + } + if hsh != theirHsh { + tracef( + CDebug, "depfix: %s/%s -> %s: hash differs", + root, fi.Name(), dep, + ) + continue + } + fields = []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: dep}, + {Name: "Hash", Value: hsh}, + } + fields = append(fields, inode.RecfileFields()...) + fieldses[len(fieldses)-1] = fields + tracef( + CDebug, "depfix: %s/%s -> %s: inode updated", + root, fi.Name(), dep, + ) + depChanged = true + } + fdDep.Close() + if !depChanged { + continue + } + redoDirChanged = true + fdDep, err = tempfile(redoDir, fi.Name()) + if err != nil { + return err + } + defer fdDep.Close() + tracef( + CDebug, "depfix: %s/%s: tmp %s", + root, fi.Name(), fdDep.Name(), + ) + w := recfile.NewWriter(fdDep) + if _, err := w.WriteFields(fieldses[0]...); err != nil { + return err + } + fieldses = fieldses[1:] + for _, fields := range fieldses { + if _, err := w.RecordStart(); err != nil { + return err + } + if _, err := w.WriteFields(fields...); err != nil { + return err + } + } + if !NoSync { + if err = fdDep.Sync(); err != nil { + return err + } + } + fdDep.Close() + if err = os.Rename(fdDep.Name(), fdDepPath); err != nil { + return err + } + tracef(CRedo, "%s", fdDepPath) + } + } + if redoDirChanged && !NoSync { + if err = syncDir(redoDir); err != nil { + return nil + } + } + return nil +} diff --git a/doc/cmds.texi b/doc/cmds.texi index 465ef65..b4e7bf5 100644 --- a/doc/cmds.texi +++ b/doc/cmds.texi @@ -75,6 +75,7 @@ And there are some maintenance and debug commands: @item redo-cleanup Removes either temporary (@option{tmp}), log files (@option{log}), or everything related to @command{goredo} (@option{full}). + @item redo-whichdo Display @file{.do} search paths for specified target (similar to @command{apenwarr/redo}): @@ -94,6 +95,7 @@ default.do ../default.o.do ../default.do @end example + @item redo-dot Dependency @url{https://en.wikipedia.org/wiki/DOT_(graph_description_language), DOT} @@ -104,4 +106,13 @@ $ redo-dot target [...] > whatever.dot $ dot -Tpng whatever.dot > whatever.png # possibly add -Gsplines=ortho @end example +@item redo-depfix + When you copy your worktree to different place, then copied files + ctime will change. And because recorded dependency information + differs from updated ctimes, out-of-date algorithm will fallback to + rereading the whole files for hash calculation, that is much slower. + If you do not want to rebuild your targets from the ground, then + @command{redo-depfix} can traverse through all dependency files and + check if they have non-altered ctime values and update them in place. + @end table diff --git a/doc/news.texi b/doc/news.texi index 475fe98..69ba830 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -7,6 +7,12 @@ @item @code{flock} locks replaced with POSIX @code{fcntl} ones. They could be more portable. +@item + @command{redo-depfix} command appeared, that traverses through all + @file{.redo} directories and their dependency files, checks if + corresponding targets has the same content but different + @code{ctime}/@code{mtime} values and rewrites dependencies with that + updated inode information. @end itemize @anchor{Release 1_21_0} diff --git a/doc/ood.texi b/doc/ood.texi index dec389f..7f00696 100644 --- a/doc/ood.texi +++ b/doc/ood.texi @@ -50,3 +50,9 @@ However GNU/Linux with @code{ext4} filesystem can easily have pretty big granularity of 10ms. @command{goredo} uses @env{$REDO_INODE_TRUST=ctime} by default. + +If you move your worktree to different place, then all @code{ctime}s +(probably @code{mtime}s if you are inaccurate) will be also changed. OOD +check will be much slower after that, because it has to fallback to +content/hash checking all the time. You can use @command{redo-depfix} +utility to rebuild dependency files. diff --git a/main.go b/main.go index 9b614d4..63de931 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ const ( CmdNameRedoStamp = "redo-stamp" CmdNameRedoTargets = "redo-targets" CmdNameRedoWhichdo = "redo-whichdo" + CmdNameRedoDepFix = "redo-depfix" ) var ( @@ -112,6 +113,7 @@ func main() { CmdNameRedoAffects, CmdNameRedoAlways, CmdNameRedoCleanup, + CmdNameRedoDepFix, CmdNameRedoDot, CmdNameRedoIfchange, CmdNameRedoIfcreate, @@ -494,6 +496,8 @@ CmdSwitch: for _, src := range srcs { fmt.Println(src) } + case CmdNameRedoDepFix: + err = depFix(Cwd) default: log.Fatalln("unknown command", cmdName) } diff --git a/ood.go b/ood.go index 39c265c..8930772 100644 --- a/ood.go +++ b/ood.go @@ -46,6 +46,8 @@ var ( OODTgts map[string]struct{} FdOODTgts *os.File FdOODTgtsLock *os.File + + ErrMissingTarget = errors.New("invalid format of .rec: missing Target") ) type TgtError struct { @@ -142,7 +144,7 @@ func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, erro for _, m := range depInfo.ifchanges { dep := m["Target"] if dep == "" { - return ood, TgtError{tgtOrig, errors.New("invalid format of .rec: missing Target")} + return ood, TgtError{tgtOrig, ErrMissingTarget} } theirInode, err := inodeFromRec(m) if err != nil { diff --git a/usage.go b/usage.go index 9d929e3..a436c46 100644 --- a/usage.go +++ b/usage.go @@ -105,13 +105,18 @@ List all currently known out-of-date targets.` d = `Usage: redo-affects target [...] List all targets that will be affected by changing the specified ones.` + case CmdNameRedoDepFix: + d = `Usage: redo-depfix + +Traverse over all .redo directories beneath and check if inode's information +(ctime/mtime) differs. Update dependency if file's content is still the same.` default: d = `Usage: goredo -symlinks goredo expects to be called through the symbolic link to it. Available commands: redo, redo-affects, redo-always, redo-cleanup, -redo-dot, redo-ifchange, redo-ifcreate, redo-log, redo-ood, -redo-sources, redo-stamp, redo-targets, redo-whichdo.` +redo-depfix, redo-dot, redo-ifchange, redo-ifcreate, redo-log, +redo-ood, redo-sources, redo-stamp, redo-targets, redo-whichdo.` } fmt.Fprintf(os.Stderr, "%s\n\nCommon options:\n", d) flag.PrintDefaults() -- 2.44.0