X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=ood.go;h=120a7de8afd554d95cb8397c01182a4db6eac99e;hb=31406bb48bdccc166adb165eecbf264c2ccf3f7a;hp=9b6ebbab03b294b2d82f648e01098aa4747b3a29;hpb=6ef14f9de82e4a61532b84b149bde0c953d58a04;p=goredo.git diff --git a/ood.go b/ood.go index 9b6ebba..120a7de 100644 --- a/ood.go +++ b/ood.go @@ -1,311 +1,277 @@ -/* -goredo -- redo implementation on pure Go -Copyright (C) 2020 Sergey Matveev - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ +// goredo -- djb's redo implementation on pure Go +// Copyright (C) 2020-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . // Out-of-date determination package main import ( + "bytes" "errors" "fmt" "io" + "io/fs" + "log" "os" - "path" - "path/filepath" "strings" - "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" ) const ( - DepTypeIfcreate = "ifcreate" - DepTypeIfchange = "ifchange" - DepTypeAlways = "always" - DepTypeStamp = "stamp" + EnvOODTgtsFd = "REDO_OOD_TGTS_FD" + EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD" ) -type TgtErr struct { - Tgt string - Err error -} +var ( + OODTgts map[string]struct{} + FdOODTgts *os.File + FdOODTgtsLock *os.File -func (e TgtErr) Unwrap() error { return e.Err } + OODCache = make(map[string]bool) + FileExistsCache = make(map[string]bool) +) -func (e TgtErr) Error() string { - return fmt.Sprintf("%s: %s", e.Tgt, e.Err) +func FileExists(p string) bool { + if exists, known := FileExistsCache[p]; known { + return exists + } + _, err := os.Stat(p) + if err == nil { + FileExistsCache[p] = true + return true + } + if errors.Is(err, fs.ErrNotExist) { + FileExistsCache[p] = false + } + return false } -func cwdMustRel(paths ...string) string { - rel, err := filepath.Rel(Cwd, path.Join(paths...)) - if err != nil { - panic(err) - } - return rel +type TgtError struct { + Tgt *Tgt + Err error } -func cwdAndTgt(tgt string) (string, string) { - cwd, tgt := path.Split(tgt) - cwd, err := filepath.Abs(cwd) - if err != nil { - panic(err) - } - return cwd, tgt +func (e TgtError) Unwrap() error { return e.Err } + +func (e TgtError) Error() string { + return fmt.Sprintf("%s: %s", e.Tgt, e.Err) } -func isSrc(cwd, tgt string) bool { - d, f := path.Split(path.Join(cwd, tgt)) - if _, err := os.Stat(path.Join(d, f)); err != nil { +func isSrc(tgt *Tgt) bool { + if !FileExists(tgt.a) { return false } - if _, err := os.Stat(path.Join(d, RedoDir, f+DepSuffix)); err == nil { + if FileExists(tgt.a + ".do") { + return false + } + if FileExists(tgt.dep) { return false } return true } -type DepInfo struct { - build string - always bool - stamp string - stampSame bool - ifcreates []string - ifchanges []map[string]string +func isOODByBuildUUID(tgt *Tgt) bool { + build, err := depBuildRead(tgt.dep) + return err != nil || build != BuildUUID } -func depRead(fdDep *os.File) (*DepInfo, error) { - r := recfile.NewReader(fdDep) - m, err := r.NextMap() - if err != nil { - return nil, err - } - depInfo := DepInfo{} - if b := m["Build"]; b == "" { - return nil, errors.New(".dep missing Build:") - } else { - depInfo.build = b +func isOOD(tgt *Tgt, level int, seen map[string]*Tgt) (bool, error) { + indent := strings.Repeat(". ", level) + tracef(CDebug, "ood: %s%s checking", indent, tgt) + ood, cached := OODCache[tgt.rel] + if cached { + tracef(CDebug, "ood: %s%s -> cached: %v", indent, tgt, ood) + return ood, nil } - for { - m, err := r.NextMap() + dep := DepCache[tgt.rel] + var err error + if dep == nil { + dep, err = depRead(tgt) if err != nil { - if err == io.EOF { - break - } - return nil, err - } - switch m["Type"] { - case DepTypeAlways: - depInfo.always = true - case DepTypeIfcreate: - dep := m["Target"] - if dep == "" { - return nil, errors.New("invalid format of .dep") + if errors.Is(err, fs.ErrNotExist) { + if isSrc(tgt) { + ood = false + tracef(CDebug, "ood: %s%s -> is source", indent, tgt) + } else { + ood = true + tracef(CDebug, "ood: %s%s -> no dep: %s", indent, tgt, tgt.dep) + } + OODCache[tgt.rel] = ood + return ood, nil } - depInfo.ifcreates = append(depInfo.ifcreates, dep) - case DepTypeIfchange: - delete(m, "Type") - depInfo.ifchanges = append(depInfo.ifchanges, m) - case DepTypeStamp: - hsh := m["Hash"] - if hsh == "" { - return nil, errors.New("invalid format of .dep") + if err != nil { + return true, TgtError{tgt, ErrLine(err)} } - depInfo.stamp = hsh - depInfo.stampSame = m["Same"] == "true" - default: - return nil, errors.New("invalid format of .dep") } + DepCache[tgt.rel] = dep } - return &depInfo, nil -} - -func rebuildStamped(cwd, tgt, depPath, stampPrev string) (bool, error) { - relTgt := cwdMustRel(cwd, tgt) - errs := make(chan error, 1) - if err := runScript(relTgt, errs, stampPrev); err != nil { - return false, err - } - if err := <-errs; !isOkRun(err) { - return false, errors.New("build failed") - } - fdDep, err := os.Open(depPath) - if err != nil { - return false, err - } - defer fdDep.Close() - depInfo, err := depRead(fdDep) - if err != nil { - return false, err - } - if depInfo.build != BuildUUID { - return false, errors.New("is not built") - } - return depInfo.stampSame, nil -} - -func formDepPath(cwd, tgt string) string { - cwd, tgt = cwdAndTgt(path.Join(cwd, tgt)) - return path.Join(cwd, RedoDir, tgt+DepSuffix) -} -func isOOD(cwd, tgtOrig string, level int, depInfo *DepInfo) (bool, error) { - indent := strings.Repeat(". ", level) - trace(CDebug, "ood: %s%s checking", indent, tgtOrig) - cwd, tgt := cwdAndTgt(path.Join(cwd, tgtOrig)) - depPath := formDepPath(cwd, tgt) - if depInfo == nil { - fdDep, err := os.Open(depPath) - if err != nil { - trace(CDebug, "ood: %s%s -> no dep: %s", indent, tgtOrig, depPath) - return true, nil - } - defer fdDep.Close() - depInfo, err = depRead(fdDep) - if err != nil { - return true, TgtErr{tgtOrig, err} - } - } - - if depInfo.build == BuildUUID { - trace(CDebug, "ood: %s%s -> already built", indent, tgtOrig) + if dep.build == BuildUUID { + tracef(CDebug, "ood: %s%s -> already built", indent, tgt) + OODCache[tgt.rel] = false return false, nil } - ood := depInfo.always - if ood { - goto StampCheck + if !FileExists(tgt.a) { + tracef(CDebug, "ood: %s%s -> non-existent", indent, tgt) + OODCache[tgt.rel] = true + return true, nil } - for _, dep := range depInfo.ifcreates { - if _, err := os.Stat(path.Join(cwd, dep)); err == nil { - trace(CDebug, "ood: %s%s -> %s created", indent, tgtOrig, dep) + for _, ifcreate := range dep.ifcreates { + if FileExists(ifcreate.a) { + tracef(CDebug, "ood: %s%s -> %s created", indent, tgt, ifcreate) ood = true - goto StampCheck + goto Done } } - for _, m := range depInfo.ifchanges { - dep := m["Target"] - theirTs := m["Ctime"] - theirHsh := m["Hash"] - if dep == "" || theirTs == "" { - return ood, TgtErr{tgtOrig, errors.New("invalid format of .dep")} + for _, ifchange := range dep.ifchanges { + tracef(CDebug, "ood: %s%s -> %s: checking", indent, tgt, ifchange.tgt) + ood, cached = OODCache[ifchange.tgt.rel] + if cached { + tracef(CDebug, "ood: %s%s -> %s: cached: %v", indent, tgt, ifchange.tgt, ood) + if ood { + goto Done + } + continue } - trace(CDebug, "ood: %s%s -> %s: checking", indent, tgtOrig, dep) - fd, err := os.Open(path.Join(cwd, dep)) + inode, err := inodeFromFileByPath(ifchange.tgt.a) if err != nil { - if os.IsNotExist(err) { - trace(CDebug, "ood: %s%s -> %s: not exists", indent, tgtOrig, dep) + if errors.Is(err, fs.ErrNotExist) { + tracef(CDebug, "ood: %s%s -> %s: not exists", indent, tgt, ifchange.tgt) ood = true - goto StampCheck + OODCache[ifchange.tgt.rel] = ood + goto Done } - return ood, TgtErr{tgtOrig, err} + return ood, TgtError{tgt, ErrLine(err)} } - defer fd.Close() - var depDepInfo *DepInfo - if !(dep == tgt || isSrc(cwd, dep)) { - trace(CDebug, "ood: %s%s -> %s: prereading .dep", indent, tgtOrig, dep) - depFdDep, err := os.Open(formDepPath(cwd, dep)) + if !bytes.Equal(inode[:8], ifchange.Inode()[:8]) { + tracef(CDebug, "ood: %s%s -> %s: size differs", indent, tgt, ifchange.tgt) + ood = true + OODCache[ifchange.tgt.rel] = ood + goto Done + } + if InodeTrust != InodeTrustNone && inode.Equals(ifchange.Inode()) { + tracef(CDebug, "ood: %s%s -> %s: same inode", indent, tgt, ifchange.tgt) + } else { + tracef(CDebug, "ood: %s%s -> %s: inode differs", indent, tgt, ifchange.tgt) + fd, err := os.Open(ifchange.tgt.a) if err != nil { - return ood, TgtErr{path.Join(tgtOrig, dep), err} + return ood, TgtError{tgt, ErrLine(err)} } - defer depFdDep.Close() - depDepInfo, err = depRead(depFdDep) + hsh, err := fileHash(fd) + fd.Close() if err != nil { - return ood, TgtErr{path.Join(tgtOrig, dep), err} + return ood, TgtError{tgt, ErrLine(err)} } - } - - if depDepInfo != nil && depDepInfo.build == BuildUUID { - trace( - CDebug, "ood: %s%s -> %s: .dep says build is same", - indent, tgtOrig, dep, - ) - if !depDepInfo.stampSame { - trace( - CDebug, "ood: %s%s -> %s: .dep says stamp is not same", - indent, tgtOrig, dep, - ) + if ifchange.Hash() != hsh { + tracef(CDebug, "ood: %s%s -> %s: hash differs", indent, tgt, ifchange.tgt) ood = true - return ood, nil + OODCache[ifchange.tgt.rel] = ood + goto Done } - trace( - CDebug, "ood: %s%s -> %s: .dep says stamp is same", - indent, tgtOrig, dep, - ) - continue + tracef(CDebug, "ood: %s%s -> %s: same hash", indent, tgt, ifchange.tgt) } - if depDepInfo == nil || !depDepInfo.always && depDepInfo.stamp == "" { - ts, err := fileCtime(fd) - if err != nil { - return ood, TgtErr{tgtOrig, err} - } - if theirTs == ts { - trace(CDebug, "ood: %s%s -> %s: same ctime", indent, tgtOrig, dep) - } else if NoHash || theirHsh == "" { - trace(CDebug, "ood: %s%s -> %s: ctime differs", indent, tgtOrig, dep) - ood = true - goto StampCheck - } else { - hsh, err := fileHash(fd) - if err != nil { - return ood, TgtErr{tgtOrig, err} - } - if theirHsh != hsh { - trace(CDebug, "ood: %s%s -> %s: hash differs", indent, tgtOrig, dep) - ood = true - goto StampCheck - } - trace(CDebug, "ood: %s%s -> %s: same hash", indent, tgtOrig, dep) - } + if ifchange.tgt.rel == tgt.rel { + tracef(CDebug, "ood: %s%s -> %s: same target", indent, tgt, ifchange.tgt) + continue } - - if dep == tgt { - trace(CDebug, "ood: %s%s -> %s: same target", indent, tgtOrig, dep) + if isSrc(ifchange.tgt) { + tracef(CDebug, "ood: %s%s -> %s: is source", indent, tgt, ifchange.tgt) + OODCache[ifchange.tgt.rel] = false continue } - if isSrc(cwd, dep) { - trace(CDebug, "ood: %s%s -> %s: is source", indent, tgtOrig, dep) + + if _, ok := seen[ifchange.tgt.rel]; ok { + tracef(CDebug, "ood: %s%s -> %s: was always built", indent, tgt, ifchange.tgt) + OODCache[ifchange.tgt.rel] = false continue } - depOod, err := isOOD(cwd, dep, level+1, depDepInfo) + depOOD, err := isOODWithTrace(ifchange.tgt, level+1, seen) if err != nil { - return ood, TgtErr{tgtOrig, err} + return ood, TgtError{tgt, err} } - if depOod { - trace(CDebug, "ood: %s%s -> %s: ood", indent, tgtOrig, dep) + if depOOD { + tracef(CDebug, "ood: %s%s -> %s: ood", indent, tgt, ifchange.tgt) ood = true - goto StampCheck + goto Done } - trace(CDebug, "ood: %s%s -> %s: !ood", indent, tgtOrig, dep) + tracef(CDebug, "ood: %s%s -> %s: !ood", indent, tgt, ifchange.tgt) } -StampCheck: - if ood && depInfo.stamp != "" { - trace(CDebug, "ood: %s%s run, because stamped", indent, tgtOrig) - stampSame, err := rebuildStamped(cwd, tgt, depPath, depInfo.stamp) - if err != nil { - return true, TgtErr{tgtOrig, err} +Done: + tracef(CDebug, "ood: %s%s: %v", indent, tgt, ood) + OODCache[tgt.rel] = ood + return ood, nil +} + +func isOODWithTrace(tgt *Tgt, level int, seen map[string]*Tgt) (bool, error) { + _, ood := OODTgts[tgt.a] + var err error + if ood { + if !isOODByBuildUUID(tgt) { + tracef(CDebug, "ood: %s%s -> already built", strings.Repeat(". ", level), tgt) + return false, nil } - trace(CDebug, "ood: %s%s -> stamp: same: %v", indent, tgtOrig, stampSame) - ood = !stampSame + tracef(CDebug, "ood: %s%s true, external decision", strings.Repeat(". ", level), tgt) + goto RecordOODTgt + } + ood, err = isOOD(tgt, level, seen) + if !ood { + return ood, err + } +RecordOODTgt: + flock := unix.Flock_t{ + Type: unix.F_WRLCK, + Whence: io.SeekStart, + } + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil { + log.Fatal(err) + } + if _, err = FdOODTgts.Seek(0, io.SeekEnd); err != nil { + log.Fatal(err) + } + if _, err := FdOODTgts.WriteString(tgt.a + "\x00"); err != nil { + log.Fatal(err) + } + flock.Type = unix.F_UNLCK + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil { + log.Fatal(err) + } + return true, nil +} + +func oodTgtsClear() { + var err error + flock := unix.Flock_t{ + Type: unix.F_WRLCK, + Whence: io.SeekStart, + } + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLKW, &flock); err != nil { + log.Fatal(err) + } + if err = FdOODTgts.Truncate(0); err != nil { + log.Fatal(err) + } + flock.Type = unix.F_UNLCK + if err = unix.FcntlFlock(FdOODTgtsLock.Fd(), unix.F_SETLK, &flock); err != nil { + log.Fatal(err) } - trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) - return ood, nil }