X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=ood.go;h=b46ea63ee8d6a076090c6255caf2c311795397a3;hb=b4eefdd675c9aef9ff8bd1089d031ee05733195b;hp=a7c2d22422443c75b3d4c84d5f876ed0e5fc483a;hpb=f15fe27971bcf37a06bb9542ea1907f11444515a;p=goredo.git diff --git a/ood.go b/ood.go index a7c2d22..b46ea63 100644 --- a/ood.go +++ b/ood.go @@ -1,6 +1,6 @@ /* -goredo -- redo implementation on pure Go -Copyright (C) 2020 Sergey Matveev +goredo -- djb's redo implementation on pure Go +Copyright (C) 2020-2023 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,14 +23,70 @@ import ( "errors" "fmt" "io" + "log" "os" "path" "path/filepath" "strings" - "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" ) +const ( + DepTypeIfcreate = "ifcreate" + DepTypeIfchange = "ifchange" + DepTypeAlways = "always" + DepTypeStamp = "stamp" + + EnvOODTgtsFd = "REDO_OOD_TGTS_FD" + EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD" +) + +var ( + OODTgts map[string]struct{} + FdOODTgts *os.File + FdOODTgtsLock *os.File + + OODCache map[string]bool = make(map[string]bool) + FileExistsCache map[string]bool = make(map[string]bool) + + ErrMissingTarget = errors.New("invalid format of .rec: missing Target") +) + +func FileExists(p string) bool { + if exists, known := FileExistsCache[p]; known { + return exists + } + _, err := os.Stat(p) + if err == nil { + FileExistsCache[p] = true + return true + } + if os.IsNotExist(err) { + FileExistsCache[p] = false + } + return false +} + +type TgtError struct { + Tgt string + Err error +} + +func (e TgtError) Unwrap() error { return e.Err } + +func (e TgtError) Error() string { + return fmt.Sprintf("%s: %s", e.Tgt, e.Err) +} + +func cwdMustRel(paths ...string) string { + rel, err := filepath.Rel(Cwd, path.Join(paths...)) + if err != nil { + panic(err) + } + return rel +} + func cwdAndTgt(tgt string) (string, string) { cwd, tgt := path.Split(tgt) cwd, err := filepath.Abs(cwd) @@ -42,185 +98,238 @@ func cwdAndTgt(tgt string) (string, string) { func isSrc(cwd, tgt string) bool { d, f := path.Split(path.Join(cwd, tgt)) - if _, err := os.Stat(path.Join(d, f)); err != nil { + if !FileExists(path.Join(d, f)) { + return false + } + if FileExists(path.Join(d, f+".do")) { return false } - if _, err := os.Stat(path.Join(d, RedoDir, f+DepSuffix)); err == nil { + if FileExists(path.Join(d, RedoDir, f+DepSuffix)) { return false } return true } -func isBuiltNow(fdDep *os.File) (bool, *recfile.Reader, error) { - r := recfile.NewReader(fdDep) - m, err := r.NextMap() +func isOODByBuildUUID(cwd, tgtOrig string) bool { + cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) + depPath := path.Join(cwd, RedoDir, tgt+DepSuffix) + fdDep, err := os.Open(depPath) if err != nil { - return false, nil, err + return true } - if m["Build"] == "" { - return false, r, errors.New("dep. missing Build:") + depInfo, err := depRead(fdDep) + fdDep.Close() + if err != nil || depInfo.build != BuildUUID { + return true } - return m["Build"] == BuildUUID, r, nil + return false } -func isOOD(cwd, tgt string, level int) (bool, error) { - tgtOrig := tgt +func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, error) { indent := strings.Repeat(". ", level) - trace(CDebug, "ood: %s%s checking", indent, tgtOrig) - cwd, tgt = cwdAndTgt(path.Join(cwd, tgt)) + tracef(CDebug, "ood: %s%s checking", indent, tgtOrig) + cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) + ood, cached := OODCache[path.Join(cwd, tgt)] + if cached { + tracef(CDebug, "ood: %s%s -> cached: %v", indent, tgtOrig, ood) + return ood, nil + } depPath := path.Join(cwd, RedoDir, tgt+DepSuffix) fdDep, err := os.Open(depPath) if err != nil { - trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) - return true, nil + if isSrc(cwd, tgt) { + ood = false + tracef(CDebug, "ood: %s%s -> is source", indent, tgtOrig) + } else { + ood = true + tracef(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) + } + OODCache[path.Join(cwd, tgt)] = ood + return ood, nil } - defer fdDep.Close() - ood := false - - builtNow, r, err := isBuiltNow(fdDep) + depInfo, err := depRead(fdDep) + fdDep.Close() if err != nil { - return true, err + return true, TgtError{tgtOrig, err} } - if builtNow { - trace(CDebug, "ood: %s%s -> already built", indent, tgtOrig) + + if depInfo.build == BuildUUID { + tracef(CDebug, "ood: %s%s -> already built", indent, tgtOrig) + OODCache[path.Join(cwd, tgt)] = false return false, nil } + if !FileExists(path.Join(cwd, tgt)) { + tracef(CDebug, "ood: %s%s -> non-existent", indent, tgtOrig) + OODCache[path.Join(cwd, tgt)] = true + return true, nil + } - var stampOur string - for { - m, err := r.NextMap() + for _, dep := range depInfo.ifcreates { + if FileExists(path.Join(cwd, dep)) { + tracef(CDebug, "ood: %s%s -> %s created", indent, tgtOrig, dep) + ood = true + goto Done + } + } + + for _, m := range depInfo.ifchanges { + dep := m["Target"] + if dep == "" { + return ood, TgtError{tgtOrig, ErrMissingTarget} + } + theirInode, err := inodeFromRec(m) if err != nil { - if err == io.EOF { - break - } - return true, err + return ood, TgtError{tgtOrig, fmt.Errorf("invalid format of .rec: %w", err)} } - switch m["Type"] { - case "always": - trace(CDebug, "ood: %s%s -> always", indent, tgtOrig) - ood = true - case "ifcreate": - theirTgt := m["Target"] - if theirTgt == "" { - return ood, errors.New("invalid format of dep." + tgtOrig) + theirHsh := m["Hash"] + tracef(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) + ood, cached = OODCache[path.Join(cwd, dep)] + if cached { + tracef(CDebug, "ood: %s%s -> %s: cached: %v", indent, tgtOrig, dep, ood) + if ood { + goto Done } - if _, err = os.Stat(path.Join(cwd, theirTgt)); err == nil { - trace(CDebug, "ood: %s%s -> created", indent, tgtOrig) + continue + } + + inode, err := inodeFromFileByPath(path.Join(cwd, dep)) + if err != nil { + if os.IsNotExist(err) { + tracef(CDebug, "ood: %s%s -> %s: not exists", indent, tgtOrig, dep) ood = true + OODCache[path.Join(cwd, dep)] = ood + goto Done } - case "ifchange": - dep := m["Target"] - theirTs := m["Ctime"] - theirHsh := m["Hash"] - if dep == "" || theirTs == "" { - return ood, errors.New("invalid format of dep." + tgtOrig) - } - trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) + return ood, TgtError{tgtOrig, err} + } + + if inode.Size != theirInode.Size { + tracef(CDebug, "ood: %s%s -> %s: size differs", indent, tgtOrig, dep) + ood = true + OODCache[path.Join(cwd, dep)] = ood + goto Done + } + if InodeTrust != InodeTrustNone && inode.Equals(theirInode) { + tracef(CDebug, "ood: %s%s -> %s: same inode", indent, tgtOrig, dep) + } else { + tracef(CDebug, "ood: %s%s -> %s: inode differs", indent, tgtOrig, dep) fd, err := os.Open(path.Join(cwd, dep)) if err != nil { - if os.IsNotExist(err) { - trace(CDebug, "ood: %s%s -> %s: not exists", indent, tgtOrig, dep) - ood = true - continue - } - return ood, err + return ood, TgtError{tgtOrig, err} } - defer fd.Close() - ts, err := fileCtime(fd) + hsh, err := fileHash(fd) + fd.Close() if err != nil { - return ood, err + return ood, TgtError{tgtOrig, err} } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) - } else if NoHash || theirHsh == "" { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) + if theirHsh != hsh { + tracef(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) ood = true - } else { - hsh, err := fileHash(fd) - if err != nil { - return ood, err - } - if theirHsh == hsh { - trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) - } else { - trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) - ood = true - } + OODCache[path.Join(cwd, dep)] = ood + goto Done } - fd.Close() - if ood { - continue - } - if dep == tgt { - trace(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep) - continue - } - if isSrc(cwd, dep) { - trace(CDebug, "ood: %s%s -> %s: is source", indent, tgtOrig, dep) - continue - } else { - depOod, err := isOOD(cwd, dep, level+1) - if depOod { - ood = true - trace(CDebug, "ood: %s%s -> %s: ood", indent, tgtOrig, dep) - } else { - trace(CDebug, "ood: %s%s -> %s: !ood", indent, tgtOrig, dep) - } - if err != nil { - return ood, err - } - } - case "stamp": - stampOur = m["Hash"] - trace(CDebug, "ood: %s%s -> stamped: %s", indent, tgtOrig, stampOur) - default: - return ood, errors.New("invalid format of dep." + tgtOrig) + tracef(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) } - } - if ood && stampOur != "" { - errs := make(chan error, 0) - trace(CDebug, "ood: %s%s running, because stamped", indent, tgtOrig) - relTgt, err := filepath.Rel(Cwd, path.Join(cwd, tgt)) - if err != nil { - return true, err - } - if err = runScript(relTgt, errs); err != nil { - return true, err + + if dep == tgt { + tracef(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep) + continue } - if err = <-errs; err != nil { - return true, err + if isSrc(cwd, dep) { + tracef(CDebug, "ood: %s%s -> %s: is source", indent, tgtOrig, dep) + OODCache[path.Join(cwd, dep)] = false + continue } - fdDep, err := os.Open(depPath) - if err != nil { - return true, err + + if _, ok := seen[cwdMustRel(cwd, dep)]; ok { + tracef(CDebug, "ood: %s%s -> %s: was always built", indent, tgtOrig, dep) + OODCache[path.Join(cwd, dep)] = false + continue } - defer fdDep.Close() - builtNow, r, err := isBuiltNow(fdDep) + + depOod, err := isOODWithTrace(cwd, dep, level+1, seen) if err != nil { - return true, err - } - if !builtNow { - return true, fmt.Errorf("%s is not built", tgtOrig) - } - var stampTheir string - for { - m, err := r.NextMap() - if err != nil { - if err == io.EOF { - break - } - return true, err - } - if m["Type"] == "stamp" { - stampTheir = m["Hash"] - break - } + return ood, TgtError{tgtOrig, err} } - trace(CDebug, "ood: %s%s -> stamp: %s %s", indent, tgtOrig, stampOur, stampTheir) - if stampOur == stampTheir { - ood = false + if depOod { + tracef(CDebug, "ood: %s%s -> %s: ood", indent, tgtOrig, dep) + ood = true + goto Done } + tracef(CDebug, "ood: %s%s -> %s: !ood", indent, tgtOrig, dep) } - trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) + +Done: + tracef(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) + OODCache[path.Join(cwd, tgt)] = ood return ood, nil } + +func isOODWithTrace( + cwd, tgtOrig string, + level int, + seen map[string]struct{}, +) (bool, error) { + p, err := filepath.Abs(path.Join(cwd, tgtOrig)) + if err != nil { + panic(err) + } + _, ood := OODTgts[p] + if ood { + if !isOODByBuildUUID(cwd, tgtOrig) { + tracef( + CDebug, + "ood: %s%s -> already built", + strings.Repeat(". ", level), tgtOrig, + ) + return false, nil + } + tracef( + CDebug, + "ood: %s%s true, external decision", + strings.Repeat(". ", level), tgtOrig, + ) + goto RecordOODTgt + } + ood, err = isOOD(cwd, tgtOrig, level, seen) + if !ood { + return ood, err + } +RecordOODTgt: + flock := unix.Flock_t{ + Type: unix.F_WRLCK, + Whence: io.SeekStart, + } + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil { + log.Fatalln(err) + } + if _, err = FdOODTgts.Seek(0, io.SeekEnd); err != nil { + log.Fatalln(err) + } + if _, err := FdOODTgts.WriteString(p + "\x00"); err != nil { + log.Fatalln(err) + } + flock.Type = unix.F_UNLCK + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil { + log.Fatalln(err) + } + return true, nil +} + +func oodTgtsClear() { + var err error + flock := unix.Flock_t{ + Type: unix.F_WRLCK, + Whence: io.SeekStart, + } + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil { + log.Fatalln(err) + } + if err = FdOODTgts.Truncate(0); err != nil { + log.Fatalln(err) + } + flock.Type = unix.F_UNLCK + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil { + log.Fatalln(err) + } +}