From 314f58ec690c7321535d6718e8d3a0ecb4cac019 Mon Sep 17 00:00:00 2001 From: Sergey Matveev Date: Mon, 15 Feb 2021 13:08:03 +0300 Subject: [PATCH] Repetitive OOD optimization --- doc/news.texi | 8 +++++++ ifchange.go | 5 +++-- main.go | 56 ++++++++++++++++++++++++++++++++++++++++++++---- ood.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++- run.go | 7 ++++++ usage.go | 2 +- 6 files changed, 129 insertions(+), 8 deletions(-) diff --git a/doc/news.texi b/doc/news.texi index 56b537d..0979623 100644 --- a/doc/news.texi +++ b/doc/news.texi @@ -1,6 +1,14 @@ @node News @unnumbered News +@anchor{Release 1.3.0} +@section Release 1.3.0 +@itemize +@item + Repetitive OOD determination optimization: pass all already known to + be OOD targets to redoing targets. +@end itemize + @anchor{Release 1.2.0} @section Release 1.2.0 @itemize diff --git a/ifchange.go b/ifchange.go index f89da48..1f2c381 100644 --- a/ifchange.go +++ b/ifchange.go @@ -139,7 +139,7 @@ RebuildDeps: jobs := 0 queueSrc = []string{} for _, tgt := range queue { - ood, err := isOOD(Cwd, tgt, 0, seen) + ood, err := isOODWithTrace(Cwd, tgt, 0, seen) if err != nil { trace(CErr, "dependant error: %s, skipping dependants", err) return nil @@ -176,6 +176,7 @@ func ifchange(tgts []string, forced, traced bool) (bool, error) { defer jsAcquire("ifchange exiting") defer Jobs.Wait() seen := buildDependants(tgts) + oodTgtsClear() trace(CDebug, "building %d targets: %v", len(tgts), tgts) jobs := 0 errs := make(chan error, len(tgts)) @@ -188,7 +189,7 @@ func ifchange(tgts []string, forced, traced bool) (bool, error) { } ood = true if !forced { - ood, err = isOOD(Cwd, tgt, 0, seen) + ood, err = isOODWithTrace(Cwd, tgt, 0, seen) if err != nil { return false, err } diff --git a/main.go b/main.go index 2007215..4de78be 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,8 @@ along with this program. If not, see . package main import ( + "bufio" + "bytes" "crypto/rand" "flag" "fmt" @@ -30,6 +32,7 @@ import ( "strconv" "go.cypherpunks.ru/recfile" + "golang.org/x/sys/unix" ) var ( @@ -159,6 +162,46 @@ func main() { } // Those are internal envs + FdOODTgts, err = ioutil.TempFile("", "ood-tgts") + if err != nil { + panic(err) + } + if err = os.Remove(FdOODTgts.Name()); err != nil { + panic(err) + } + FdOODTgtsLock, err = ioutil.TempFile("", "ood-tgts.lock") + if err != nil { + panic(err) + } + if err = os.Remove(FdOODTgtsLock.Name()); err != nil { + panic(err) + } + + if v := os.Getenv(EnvOODTgtsFd); v != "" { + fd := mustParseFd(v, EnvOODTgtsFd) + fdLock := mustParseFd(v, EnvOODTgtsLockFd) + if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if _, err = fd.Seek(0, os.SEEK_SET); err != nil { + panic(err) + } + tgtsRaw, err := ioutil.ReadAll(bufio.NewReader(fd)) + if err != nil { + panic(err) + } + unix.Flock(int(fdLock.Fd()), unix.LOCK_UN) + OODTgts = make(map[string]struct{}) + for _, tgtRaw := range bytes.Split(tgtsRaw, []byte{0}) { + t := string(tgtRaw) + if t == "" { + continue + } + OODTgts[t] = struct{}{} + trace(CDebug, "ood: known to be: %s", t) + } + } + StderrPrefix = os.Getenv(EnvStderrPrefix) if v := os.Getenv(EnvLevel); v != "" { Level, err = strconv.Atoi(v) @@ -169,12 +212,14 @@ func main() { panic("negative " + EnvLevel) } } + var fdDep *os.File if v := os.Getenv(EnvDepFd); v != "" { fdDep = mustParseFd(v, EnvDepFd) } - BuildUUID = os.Getenv(EnvBuildUUID) + tgts := flag.Args() + BuildUUID = os.Getenv(EnvBuildUUID) if BuildUUID == "" { raw := new([16]byte) if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil { @@ -189,6 +234,7 @@ func main() { tgts = []string{"all"} } } + statusInit() for i, tgt := range tgts { @@ -264,17 +310,19 @@ CmdSwitch: log.Fatalln("single target expected") } var fdTmp *os.File - fdTmp, err = ioutil.TempFile("", "") + fdTmp, err = ioutil.TempFile("", "whichdo") if err != nil { break } - os.Remove(fdTmp.Name()) + if err = os.Remove(fdTmp.Name()); err != nil { + break + } cwd, tgt := cwdAndTgt(tgts[0]) doFile, upLevels, err := findDo(fdTmp, cwd, tgt) if err != nil { break } - _, err = fdTmp.Seek(0, 0) + _, err = fdTmp.Seek(0, os.SEEK_SET) if err != nil { break } diff --git a/ood.go b/ood.go index 0552e20..cad1ce3 100644 --- a/ood.go +++ b/ood.go @@ -26,6 +26,8 @@ import ( "path" "path/filepath" "strings" + + "golang.org/x/sys/unix" ) const ( @@ -33,6 +35,15 @@ const ( DepTypeIfchange = "ifchange" DepTypeAlways = "always" DepTypeStamp = "stamp" + + EnvOODTgtsFd = "REDO_OOD_TGTS_FD" + EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD" +) + +var ( + OODTgts map[string]struct{} + FdOODTgts *os.File + FdOODTgtsLock *os.File ) type TgtErr struct { @@ -170,7 +181,7 @@ func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, erro continue } - depOod, err := isOOD(cwd, dep, level+1, seen) + depOod, err := isOODWithTrace(cwd, dep, level+1, seen) if err != nil { return ood, TgtErr{tgtOrig, err} } @@ -186,3 +197,49 @@ Done: trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood) return ood, nil } + +func isOODWithTrace( + cwd, tgtOrig string, + level int, + seen map[string]struct{}, +) (bool, error) { + p, err := filepath.Abs(path.Join(cwd, tgtOrig)) + if err != nil { + panic(err) + } + _, ood := OODTgts[p] + if ood { + trace( + CDebug, + "ood: %s%s true, external decision", + strings.Repeat(". ", level), tgtOrig, + ) + goto RecordOODTgt + } + ood, err = isOOD(cwd, tgtOrig, level, seen) + if !ood { + return ood, err + } +RecordOODTgt: + if err = unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if _, err = FdOODTgts.Seek(0, os.SEEK_END); err != nil { + panic(err) + } + if _, err := FdOODTgts.WriteString(p + "\x00"); err != nil { + panic(err) + } + unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN) + return true, nil +} + +func oodTgtsClear() { + if err := unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil { + panic(err) + } + if err := FdOODTgts.Truncate(0); err != nil { + panic(err) + } + unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN) +} diff --git a/run.go b/run.go index af17eaa..af71b3b 100644 --- a/run.go +++ b/run.go @@ -345,6 +345,13 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { )) fdNum := 0 + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum)) + fdNum++ + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum)) + fdNum++ + if FdStatus == nil { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd)) } else { diff --git a/usage.go b/usage.go index affd454..a8bb28b 100644 --- a/usage.go +++ b/usage.go @@ -26,7 +26,7 @@ import ( ) const ( - Version = "1.2.0" + Version = "1.3.0" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 3 of the License. -- 2.44.0