/*
-goredo -- redo implementation on pure Go
-Copyright (C) 2020 Sergey Matveev <stargrave@stargrave.org>
+goredo -- djb's redo implementation on pure Go
+Copyright (C) 2020-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
"errors"
"fmt"
"io"
+ "io/fs"
+ "log"
"os"
- "path"
- "path/filepath"
"strings"
- "go.cypherpunks.ru/recfile"
+ "golang.org/x/sys/unix"
)
-func cwdAndTgt(tgt string) (string, string) {
- cwd, tgt := path.Split(tgt)
- cwd, err := filepath.Abs(cwd)
- if err != nil {
- panic(err)
+const (
+ DepTypeIfcreate = "ifcreate"
+ DepTypeIfchange = "ifchange"
+ DepTypeAlways = "always"
+ DepTypeStamp = "stamp"
+
+ EnvOODTgtsFd = "REDO_OOD_TGTS_FD"
+ EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD"
+)
+
+var (
+ OODTgts map[string]struct{}
+ FdOODTgts *os.File
+ FdOODTgtsLock *os.File
+
+ OODCache = make(map[string]bool)
+ FileExistsCache = make(map[string]bool)
+ DepInfoCache = make(map[string]*DepInfo)
+
+ ErrMissingTarget = errors.New("invalid format of .rec: missing Target")
+)
+
+func FileExists(p string) bool {
+ if exists, known := FileExistsCache[p]; known {
+ return exists
+ }
+ _, err := os.Stat(p)
+ if err == nil {
+ FileExistsCache[p] = true
+ return true
+ }
+ if errors.Is(err, fs.ErrNotExist) {
+ FileExistsCache[p] = false
}
- return cwd, tgt
+ return false
+}
+
+type TgtError struct {
+ Tgt *Tgt
+ Err error
+}
+
+func (e TgtError) Unwrap() error { return e.Err }
+
+func (e TgtError) Error() string {
+ return fmt.Sprintf("%s: %s", e.Tgt, e.Err)
}
-func isSrc(cwd, tgt string) bool {
- d, f := path.Split(path.Join(cwd, tgt))
- if _, err := os.Stat(path.Join(d, f)); err != nil {
+func isSrc(tgt *Tgt) bool {
+ if !FileExists(tgt.a) {
+ return false
+ }
+ if FileExists(tgt.a + ".do") {
return false
}
- if _, err := os.Stat(path.Join(d, RedoDir, f+DepSuffix)); err == nil {
+ if FileExists(tgt.Dep()) {
return false
}
return true
}
-func isBuiltNow(fdDep *os.File) (bool, *recfile.Reader, error) {
- r := recfile.NewReader(fdDep)
- m, err := r.NextMap()
- if err != nil {
- return false, nil, err
- }
- if m["Build"] == "" {
- return false, r, errors.New("dep. missing Build:")
- }
- return m["Build"] == BuildUUID, r, nil
+func isOODByBuildUUID(tgt *Tgt) bool {
+ build, err := depReadBuild(tgt.Dep())
+ return err != nil || build != BuildUUID
}
-func isOOD(cwd, tgt string, level int) (bool, error) {
- tgtOrig := tgt
+func isOOD(tgt *Tgt, level int, seen map[string]*Tgt) (bool, error) {
indent := strings.Repeat(". ", level)
- trace(CDebug, "ood: %s%s checking", indent, tgtOrig)
- cwd, tgt = cwdAndTgt(path.Join(cwd, tgt))
- depPath := path.Join(cwd, RedoDir, tgt+DepSuffix)
- fdDep, err := os.Open(depPath)
- if err != nil {
- trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath)
- return true, nil
+ tracef(CDebug, "ood: %s%s checking", indent, tgt)
+ ood, cached := OODCache[tgt.a]
+ if cached {
+ tracef(CDebug, "ood: %s%s -> cached: %v", indent, tgt, ood)
+ return ood, nil
}
- defer fdDep.Close()
- ood := false
-
- builtNow, r, err := isBuiltNow(fdDep)
- if err != nil {
- return true, err
+ depInfo := DepInfoCache[tgt.Dep()]
+ var err error
+ if depInfo == nil {
+ depInfo, err = depRead(tgt)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ if isSrc(tgt) {
+ ood = false
+ tracef(CDebug, "ood: %s%s -> is source", indent, tgt)
+ } else {
+ ood = true
+ tracef(CDebug, "ood: %s%s -> no dep: %s", indent, tgt, tgt.Dep())
+ }
+ OODCache[tgt.a] = ood
+ return ood, nil
+ }
+ if err != nil {
+ return true, TgtError{tgt, ErrLine(err)}
+ }
+ }
+ DepInfoCache[tgt.Dep()] = depInfo
}
- if builtNow {
- trace(CDebug, "ood: %s%s -> already built", indent, tgtOrig)
+
+ if depInfo.build == BuildUUID {
+ tracef(CDebug, "ood: %s%s -> already built", indent, tgt)
+ OODCache[tgt.a] = false
return false, nil
}
+ if !FileExists(tgt.a) {
+ tracef(CDebug, "ood: %s%s -> non-existent", indent, tgt)
+ OODCache[tgt.a] = true
+ return true, nil
+ }
- var stampOur string
- for {
- m, err := r.NextMap()
- if err != nil {
- if err == io.EOF {
- break
- }
- return true, err
- }
- switch m["Type"] {
- case "always":
- trace(CDebug, "ood: %s%s -> always", indent, tgtOrig)
+ for _, dep := range depInfo.ifcreates {
+ if FileExists(dep.a) {
+ tracef(CDebug, "ood: %s%s -> %s created", indent, tgt, dep)
ood = true
- case "ifcreate":
- theirTgt := m["Target"]
- if theirTgt == "" {
- return ood, errors.New("invalid format of dep." + tgtOrig)
+ goto Done
+ }
+ }
+
+ for _, dep := range depInfo.ifchanges {
+ tracef(CDebug, "ood: %s%s -> %s: checking", indent, tgt, dep.tgt)
+ ood, cached = OODCache[dep.tgt.a]
+ if cached {
+ tracef(CDebug, "ood: %s%s -> %s: cached: %v", indent, tgt, dep.tgt, ood)
+ if ood {
+ goto Done
}
- if _, err = os.Stat(path.Join(cwd, theirTgt)); err == nil {
- trace(CDebug, "ood: %s%s -> created", indent, tgtOrig)
+ continue
+ }
+
+ inode, err := inodeFromFileByPath(dep.tgt.a)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ tracef(CDebug, "ood: %s%s -> %s: not exists", indent, tgt, dep.tgt)
ood = true
+ OODCache[dep.tgt.a] = ood
+ goto Done
}
- case "ifchange":
- dep := m["Target"]
- theirTs := m["Ctime"]
- theirHsh := m["Hash"]
- if dep == "" || theirTs == "" {
- return ood, errors.New("invalid format of dep." + tgtOrig)
- }
- trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep)
- fd, err := os.Open(path.Join(cwd, dep))
+ return ood, TgtError{tgt, ErrLine(err)}
+ }
+
+ if inode.Size != dep.inode.Size {
+ tracef(CDebug, "ood: %s%s -> %s: size differs", indent, tgt, dep.tgt)
+ ood = true
+ OODCache[dep.tgt.a] = ood
+ goto Done
+ }
+ if InodeTrust != InodeTrustNone && inode.Equals(dep.inode) {
+ tracef(CDebug, "ood: %s%s -> %s: same inode", indent, tgt, dep.tgt)
+ } else {
+ tracef(CDebug, "ood: %s%s -> %s: inode differs", indent, tgt, dep.tgt)
+ fd, err := os.Open(dep.tgt.a)
if err != nil {
- if os.IsNotExist(err) {
- trace(CDebug, "ood: %s%s -> %s: not exists", indent, tgtOrig, dep)
- ood = true
- continue
- }
- return ood, err
+ return ood, TgtError{tgt, ErrLine(err)}
}
- defer fd.Close()
- ts, err := fileCtime(fd)
+ hsh, err := fileHash(fd)
+ fd.Close()
if err != nil {
- return ood, err
+ return ood, TgtError{tgt, ErrLine(err)}
}
- if theirTs == ts {
- trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep)
- } else if NoHash || theirHsh == "" {
- trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep)
+ if dep.hash != hsh {
+ tracef(CDebug, "ood: %s%s -> %s: hash differs", indent, tgt, dep.tgt)
ood = true
- } else {
- hsh, err := fileHash(fd)
- if err != nil {
- return ood, err
- }
- if theirHsh == hsh {
- trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep)
- } else {
- trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep)
- ood = true
- }
+ OODCache[dep.tgt.a] = ood
+ goto Done
}
- fd.Close()
- if ood {
- continue
- }
- if dep == tgt {
- trace(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep)
- continue
- }
- if isSrc(cwd, dep) {
- trace(CDebug, "ood: %s%s -> %s: is source", indent, tgtOrig, dep)
- continue
- } else {
- depOod, err := isOOD(cwd, dep, level+1)
- if depOod {
- ood = true
- trace(CDebug, "ood: %s%s -> %s: ood", indent, tgtOrig, dep)
- } else {
- trace(CDebug, "ood: %s%s -> %s: !ood", indent, tgtOrig, dep)
- }
- if err != nil {
- return ood, err
- }
- }
- case "stamp":
- stampOur = m["Hash"]
- trace(CDebug, "ood: %s%s -> stamped: %s", indent, tgtOrig, stampOur)
- default:
- return ood, errors.New("invalid format of dep." + tgtOrig)
- }
- }
- if ood && stampOur != "" {
- errs := make(chan error, 0)
- trace(CDebug, "ood: %s%s running, because stamped", indent, tgtOrig)
- relTgt, err := filepath.Rel(Cwd, path.Join(cwd, tgt))
- if err != nil {
- return true, err
+ tracef(CDebug, "ood: %s%s -> %s: same hash", indent, tgt, dep.tgt)
}
- if err = runScript(relTgt, errs); err != nil {
- return true, err
+
+ if dep.tgt.a == tgt.a {
+ tracef(CDebug, "ood: %s%s -> %s: same target", indent, tgt, dep.tgt)
+ continue
}
- if err = <-errs; err != nil {
- return true, err
+ if isSrc(dep.tgt) {
+ tracef(CDebug, "ood: %s%s -> %s: is source", indent, tgt, dep.tgt)
+ OODCache[dep.tgt.a] = false
+ continue
}
- fdDep, err := os.Open(depPath)
- if err != nil {
- return true, err
+
+ if _, ok := seen[dep.tgt.a]; ok {
+ tracef(CDebug, "ood: %s%s -> %s: was always built", indent, tgt, dep.tgt)
+ OODCache[dep.tgt.a] = false
+ continue
}
- defer fdDep.Close()
- builtNow, r, err := isBuiltNow(fdDep)
+
+ depOOD, err := isOODWithTrace(dep.tgt, level+1, seen)
if err != nil {
- return true, err
- }
- if !builtNow {
- return true, fmt.Errorf("%s is not built", tgtOrig)
+ return ood, TgtError{tgt, err}
}
- var stampTheir string
- for {
- m, err := r.NextMap()
- if err != nil {
- if err == io.EOF {
- break
- }
- return true, err
- }
- if m["Type"] == "stamp" {
- stampTheir = m["Hash"]
- break
- }
- }
- trace(CDebug, "ood: %s%s -> stamp: %s %s", indent, tgtOrig, stampOur, stampTheir)
- if stampOur == stampTheir {
- ood = false
+ if depOOD {
+ tracef(CDebug, "ood: %s%s -> %s: ood", indent, tgt, dep.tgt)
+ ood = true
+ goto Done
}
+ tracef(CDebug, "ood: %s%s -> %s: !ood", indent, tgt, dep.tgt)
}
- trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood)
+
+Done:
+ tracef(CDebug, "ood: %s%s: %v", indent, tgt, ood)
+ OODCache[tgt.a] = ood
return ood, nil
}
+
+func isOODWithTrace(tgt *Tgt, level int, seen map[string]*Tgt) (bool, error) {
+ _, ood := OODTgts[tgt.a]
+ var err error
+ if ood {
+ if !isOODByBuildUUID(tgt) {
+ tracef(CDebug, "ood: %s%s -> already built", strings.Repeat(". ", level), tgt)
+ return false, nil
+ }
+ tracef(CDebug, "ood: %s%s true, external decision", strings.Repeat(". ", level), tgt)
+ goto RecordOODTgt
+ }
+ ood, err = isOOD(tgt, level, seen)
+ if !ood {
+ return ood, err
+ }
+RecordOODTgt:
+ flock := unix.Flock_t{
+ Type: unix.F_WRLCK,
+ Whence: io.SeekStart,
+ }
+ if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil {
+ log.Fatal(err)
+ }
+ if _, err = FdOODTgts.Seek(0, io.SeekEnd); err != nil {
+ log.Fatal(err)
+ }
+ if _, err := FdOODTgts.WriteString(tgt.a + "\x00"); err != nil {
+ log.Fatal(err)
+ }
+ flock.Type = unix.F_UNLCK
+ if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ log.Fatal(err)
+ }
+ return true, nil
+}
+
+func oodTgtsClear() {
+ var err error
+ flock := unix.Flock_t{
+ Type: unix.F_WRLCK,
+ Whence: io.SeekStart,
+ }
+ if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil {
+ log.Fatal(err)
+ }
+ if err = FdOODTgts.Truncate(0); err != nil {
+ log.Fatal(err)
+ }
+ flock.Type = unix.F_UNLCK
+ if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil {
+ log.Fatal(err)
+ }
+}