From: Sergey Matveev Date: Sat, 16 Jan 2021 17:57:04 +0000 (+0300) Subject: More general Inode information tracking, explicit size check X-Git-Tag: v1.0.0~1 X-Git-Url: http://www.git.cypherpunks.ru/?p=goredo.git;a=commitdiff_plain;h=ebd96d2c56c742d9d00bccb9faee5fc1a7db664a More general Inode information tracking, explicit size check --- diff --git a/dep.go b/dep.go index d7c874a..55613c7 100644 --- a/dep.go +++ b/dep.go @@ -23,14 +23,12 @@ import ( "bufio" "encoding/hex" "errors" - "fmt" "io" "os" "path" "path/filepath" "go.cypherpunks.ru/recfile" - "golang.org/x/sys/unix" "lukechampine.com/blake3" ) @@ -75,15 +73,6 @@ func stamp(fdDep, src *os.File) error { ) } -func fileCtime(fd *os.File) (string, error) { - var stat unix.Stat_t - if err := unix.Fstat(int(fd.Fd()), &stat); err != nil { - return "", err - } - sec, nsec := stat.Ctim.Unix() - return fmt.Sprintf("%d.%d", sec, nsec), nil -} - func fileHash(fd *os.File) (string, error) { h := blake3.New(32, nil) if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { @@ -106,7 +95,7 @@ func writeDep(fdDep *os.File, cwd, tgt string) error { if fi.IsDir() { return nil } - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) if err != nil { return err } @@ -114,13 +103,13 @@ func writeDep(fdDep *os.File, cwd, tgt string) error { if err != nil { return err } - return recfileWrite( - fdDep, - recfile.Field{Name: "Type", Value: DepTypeIfchange}, - recfile.Field{Name: "Target", Value: tgt}, - recfile.Field{Name: "Ctime", Value: ts}, - recfile.Field{Name: "Hash", Value: hsh}, - ) + fields := []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: tgt}, + {Name: "Hash", Value: hsh}, + } + fields = append(fields, inode.RecfileFields()...) + return recfileWrite(fdDep, fields...) } func writeDeps(fdDep *os.File, tgts []string) (err error) { @@ -142,12 +131,13 @@ func writeDeps(fdDep *os.File, tgts []string) (err error) { err = writeDep(fdDep, tgtDir, tgtRel) } else { trace(CDebug, "ifchange: %s <- %s (unexisting)", fdDep.Name(), tgtRel) - err = recfileWrite( - fdDep, - recfile.Field{Name: "Type", Value: DepTypeIfchange}, - recfile.Field{Name: "Target", Value: tgtRel}, - recfile.Field{Name: "Ctime", Value: "0.0"}, - ) + fields := []recfile.Field{ + {Name: "Type", Value: DepTypeIfchange}, + {Name: "Target", Value: tgtRel}, + } + inodeDummy := Inode{} + fields = append(fields, inodeDummy.RecfileFields()...) + err = recfileWrite(fdDep, fields...) } } return diff --git a/dep.rec b/dep.rec index f0f6ced..9f41434 100644 --- a/dep.rec +++ b/dep.rec @@ -6,8 +6,10 @@ %rec: Dependency %doc: Dependency information %mandatory: Type -%allowed: Target Ctime Hash -%unique: Type Target Ctime Hash +%allowed: Target Size CtimeSec CtimeNsec Hash +%unique: Type Target Size CtimeSec CtimeNsec Hash %type: Type enum ifcreate ifchange always stamp -%type: Ctime regexp /[0-9]+\.[0-9]+/ +%type: Size int +%type: CtimeSec int +%type: CtimeNsec int %type: Hash regexp /[0-9a-f]{64}/ diff --git a/doc/features.texi b/doc/features.texi index 9145312..2c5141e 100644 --- a/doc/features.texi +++ b/doc/features.texi @@ -12,7 +12,8 @@ @end itemize @item targets, dependency information and their directories are explicitly synced (can be disabled, should work faster) -@item file's change is detected by comparing its @code{ctime} and BLAKE3 hash +@item file's change is detected by comparing its size, @code{ctime} + and BLAKE3 hash @item files creation is @code{umask}-friendly (unlike @code{mkstemp()} used in @command{redo-c}) @item parallel build with jobs limit, optionally in infinite mode diff --git a/doc/news.texi b/doc/news.texi index 5484635..bc4322e 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -1,9 +1,12 @@ @node News @unnumbered News -@anchor{Release 0.13.0} -@section Release 0.13.0 +@anchor{Release 1.0.0} +@section Release 1.0.0 @itemize +@item + @code{Size} is stored in the state, for faster OOD detection. + Previous @command{goredo} state files won't work. @item @command{redo-whichdo} resembles @code{apenwarr/redo}'s one behaviour more. @end itemize diff --git a/doc/state.texi b/doc/state.texi index 5313ae1..fa7c28c 100644 --- a/doc/state.texi +++ b/doc/state.texi @@ -15,7 +15,9 @@ Target: foo.o.do Type: ifchange Target: default.o.do -Ctime: 1605721341.253305000 +Size: 123 +CtimeSec: 1605721341 +CtimeNsec: 253305000 Hash: f4929732f96f11e6d4ebe94536b5edef426d00ed0146853e37a87f4295e18eda Type: always diff --git a/inode.go b/inode.go new file mode 100644 index 0000000..81e77bb --- /dev/null +++ b/inode.go @@ -0,0 +1,94 @@ +/* +goredo -- redo implementation on pure Go +Copyright (C) 2020-2021 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Inode metainformation + +package main + +import ( + "errors" + "os" + "strconv" + + "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" +) + +type Inode struct { + Size int64 + CtimeSec int64 + CtimeNsec int64 +} + +func (our *Inode) Equals(their *Inode) bool { + return (our.Size == their.Size) && + (our.CtimeSec == their.CtimeSec) && + (our.CtimeNsec == their.CtimeNsec) +} + +func (inode *Inode) RecfileFields() []recfile.Field { + return []recfile.Field{ + {Name: "Size", Value: strconv.FormatInt(inode.Size, 10)}, + {Name: "CtimeSec", Value: strconv.FormatInt(inode.CtimeSec, 10)}, + {Name: "CtimeNsec", Value: strconv.FormatInt(inode.CtimeNsec, 10)}, + } +} + +func inodeFromFile(fd *os.File) (*Inode, error) { + var fi os.FileInfo + fi, err := fd.Stat() + if err != nil { + return nil, err + } + var stat unix.Stat_t + err = unix.Fstat(int(fd.Fd()), &stat) + if err != nil { + return nil, err + } + sec, nsec := stat.Ctim.Unix() + return &Inode{Size: fi.Size(), CtimeSec: sec, CtimeNsec: nsec}, nil +} + +func inodeFromRec(m map[string]string) (*Inode, error) { + size := m["Size"] + ctimeSec := m["CtimeSec"] + ctimeNsec := m["CtimeNsec"] + if size == "" { + return nil, errors.New("Size is missing") + } + if ctimeSec == "" { + return nil, errors.New("CtimeSec is missing") + } + if ctimeNsec == "" { + return nil, errors.New("CtimeNsec is missing") + } + inode := Inode{} + var err error + inode.Size, err = strconv.ParseInt(size, 10, 64) + if err != nil { + return nil, err + } + inode.CtimeSec, err = strconv.ParseInt(ctimeSec, 10, 64) + if err != nil { + return nil, err + } + inode.CtimeNsec, err = strconv.ParseInt(ctimeNsec, 10, 64) + if err != nil { + return nil, err + } + return &inode, nil +} diff --git a/ood.go b/ood.go index 7f2adde..e75dd43 100644 --- a/ood.go +++ b/ood.go @@ -109,11 +109,14 @@ func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, erro for _, m := range depInfo.ifchanges { dep := m["Target"] - theirTs := m["Ctime"] - theirHsh := m["Hash"] - if dep == "" || theirTs == "" { - return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} + if dep == "" { + return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep: missing Target")} + } + theirInode, err := inodeFromRec(m) + if err != nil { + return ood, TgtErr{tgtOrig, fmt.Errorf("invalid format of .dep: %v", err)} } + theirHsh := m["Hash"] trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) fd, err := os.Open(path.Join(cwd, dep)) @@ -127,14 +130,19 @@ func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, erro } defer fd.Close() - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) if err != nil { return ood, TgtErr{tgtOrig, err} } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) + if inode.Size != theirInode.Size { + trace(CDebug, "ood: %s%s -> %s: size differs", indent, tgtOrig, dep) + ood = true + goto Done + } + if inode.Equals(theirInode) { + trace(CDebug, "ood: %s%s -> %s: same inode", indent, tgtOrig, dep) } else { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) + trace(CDebug, "ood: %s%s -> %s: inode differs", indent, tgtOrig, dep) hsh, err := fileHash(fd) if err != nil { return ood, TgtErr{tgtOrig, err} diff --git a/run.go b/run.go index d8e439d..1dc937d 100644 --- a/run.go +++ b/run.go @@ -101,24 +101,24 @@ func mkdirs(pth string) error { return os.MkdirAll(pth, os.FileMode(0777)) } -func isModified(cwd, redoDir, tgt string) (bool, string, error) { +func isModified(cwd, redoDir, tgt string) (bool, *Inode, error) { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err } defer fdDep.Close() r := recfile.NewReader(fdDep) - var ourTs string + var ourInode *Inode for { m, err := r.NextMap() if err != nil { if err == io.EOF { break } - return false, "", err + return false, nil, err } if m["Target"] != tgt { continue @@ -126,21 +126,25 @@ func isModified(cwd, redoDir, tgt string) (bool, string, error) { fd, err := os.Open(path.Join(cwd, tgt)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err } - defer fd.Close() - ourTs, err = fileCtime(fd) + ourInode, err = inodeFromFile(fd) + fd.Close() if err != nil { - return false, "", err + return false, nil, err } - if ourTs != m["Ctime"] { - return true, ourTs, nil + theirInode, err := inodeFromRec(m) + if err != nil { + return false, nil, err + } + if !ourInode.Equals(theirInode) { + return true, ourInode, nil } break } - return false, ourTs, nil + return false, ourInode, nil } func syncDir(dir string) error { @@ -222,7 +226,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } // Check if target is not modified externally - modified, tsPrev, err := isModified(cwd, redoDir, tgt) + modified, inodePrev, err := isModified(cwd, redoDir, tgt) if err != nil { lockRelease() return TgtErr{tgtOrig, err} @@ -462,11 +466,11 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } // Was $1 touched? - if tsPrev != "" { + if inodePrev != nil { if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { - ts, err := fileCtime(fd) + inode, err := inodeFromFile(fd) fd.Close() - if err == nil && ts != tsPrev { + if err == nil && !inode.Equals(inodePrev) { runErr.Err = errors.New("$1 was explicitly touched") errs <- runErr fd.Close() diff --git a/usage.go b/usage.go index 8fa4aaa..ce838b4 100644 --- a/usage.go +++ b/usage.go @@ -26,7 +26,7 @@ import ( ) const ( - Version = "0.13.0" + Version = "1.0.0" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License.