Repetitive OOD optimization

diff --git a/doc/news.texi b/doc/news.texi
index 56b537d7d81f9b62f88fe3d13ba16948a2a575fe..0979623eca341c85625e192ba84198e92c95f0f5 100644 (file)
--- a/doc/news.texi
+++ b/doc/news.texi
@@ -1,6 +1,14 @@
 @node News
 @unnumbered News
 
 @node News
 @unnumbered News
 

+@anchor{Release 1.3.0}
+@section Release 1.3.0
+@itemize
+@item
+    Repetitive OOD determination optimization: pass all already known to
+    be OOD targets to redoing targets.
+@end itemize
+

 @anchor{Release 1.2.0}
 @section Release 1.2.0
 @itemize
 @anchor{Release 1.2.0}
 @section Release 1.2.0
 @itemize

diff --git a/ifchange.go b/ifchange.go
index f89da48fc7d1dbe9b733c1e679fd8a4044760aa7..1f2c38194934da44475e97e039fb73601536a5a0 100644 (file)
--- a/ifchange.go
+++ b/ifchange.go
@@ -139,7 +139,7 @@ RebuildDeps:
        jobs := 0
        queueSrc = []string{}
        for _, tgt := range queue {
        jobs := 0
        queueSrc = []string{}
        for _, tgt := range queue {

-               ood, err := isOOD(Cwd, tgt, 0, seen)
+               ood, err := isOODWithTrace(Cwd, tgt, 0, seen)

                if err != nil {
                        trace(CErr, "dependant error: %s, skipping dependants", err)
                        return nil
                if err != nil {
                        trace(CErr, "dependant error: %s, skipping dependants", err)
                        return nil


@@ -176,6 +176,7 @@ func ifchange(tgts []string, forced, traced bool) (bool, error) {
        defer jsAcquire("ifchange exiting")
        defer Jobs.Wait()
        seen := buildDependants(tgts)
        defer jsAcquire("ifchange exiting")
        defer Jobs.Wait()
        seen := buildDependants(tgts)

+       oodTgtsClear()

        trace(CDebug, "building %d targets: %v", len(tgts), tgts)
        jobs := 0
        errs := make(chan error, len(tgts))
        trace(CDebug, "building %d targets: %v", len(tgts), tgts)
        jobs := 0
        errs := make(chan error, len(tgts))


@@ -188,7 +189,7 @@ func ifchange(tgts []string, forced, traced bool) (bool, error) {
                }
                ood = true
                if !forced {
                }
                ood = true
                if !forced {

-                       ood, err = isOOD(Cwd, tgt, 0, seen)
+                       ood, err = isOODWithTrace(Cwd, tgt, 0, seen)

                        if err != nil {
                                return false, err
                        }
                        if err != nil {
                                return false, err
                        }

diff --git a/main.go b/main.go
index 20072155f67c423ac294d82014af923f90f5280d..4de78be52fa786830cfbb92862c1c495cd59140a 100644 (file)
--- a/main.go
+++ b/main.go
@@ -18,6 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package main
 
 import (
 package main
 
 import (

+       "bufio"
+       "bytes"

        "crypto/rand"
        "flag"
        "fmt"
        "crypto/rand"
        "flag"
        "fmt"


@@ -30,6 +32,7 @@ import (
        "strconv"
 
        "go.cypherpunks.ru/recfile"
        "strconv"
 
        "go.cypherpunks.ru/recfile"

+       "golang.org/x/sys/unix"

 )
 
 var (
 )
 
 var (


@@ -159,6 +162,46 @@ func main() {
        }
 
        // Those are internal envs
        }
 
        // Those are internal envs

+       FdOODTgts, err = ioutil.TempFile("", "ood-tgts")
+       if err != nil {
+               panic(err)
+       }
+       if err = os.Remove(FdOODTgts.Name()); err != nil {
+               panic(err)
+       }
+       FdOODTgtsLock, err = ioutil.TempFile("", "ood-tgts.lock")
+       if err != nil {
+               panic(err)
+       }
+       if err = os.Remove(FdOODTgtsLock.Name()); err != nil {
+               panic(err)
+       }
+
+       if v := os.Getenv(EnvOODTgtsFd); v != "" {
+               fd := mustParseFd(v, EnvOODTgtsFd)
+               fdLock := mustParseFd(v, EnvOODTgtsLockFd)
+               if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX); err != nil {
+                       panic(err)
+               }
+               if _, err = fd.Seek(0, os.SEEK_SET); err != nil {
+                       panic(err)
+               }
+               tgtsRaw, err := ioutil.ReadAll(bufio.NewReader(fd))
+               if err != nil {
+                       panic(err)
+               }
+               unix.Flock(int(fdLock.Fd()), unix.LOCK_UN)
+               OODTgts = make(map[string]struct{})
+               for _, tgtRaw := range bytes.Split(tgtsRaw, []byte{0}) {
+                       t := string(tgtRaw)
+                       if t == "" {
+                               continue
+                       }
+                       OODTgts[t] = struct{}{}
+                       trace(CDebug, "ood: known to be: %s", t)
+               }
+       }
+

        StderrPrefix = os.Getenv(EnvStderrPrefix)
        if v := os.Getenv(EnvLevel); v != "" {
                Level, err = strconv.Atoi(v)
        StderrPrefix = os.Getenv(EnvStderrPrefix)
        if v := os.Getenv(EnvLevel); v != "" {
                Level, err = strconv.Atoi(v)


@@ -169,12 +212,14 @@ func main() {
                        panic("negative " + EnvLevel)
                }
        }
                        panic("negative " + EnvLevel)
                }
        }

+

        var fdDep *os.File
        if v := os.Getenv(EnvDepFd); v != "" {
                fdDep = mustParseFd(v, EnvDepFd)
        }
        var fdDep *os.File
        if v := os.Getenv(EnvDepFd); v != "" {
                fdDep = mustParseFd(v, EnvDepFd)
        }

-       BuildUUID = os.Getenv(EnvBuildUUID)
+

        tgts := flag.Args()
        tgts := flag.Args()

+       BuildUUID = os.Getenv(EnvBuildUUID)

        if BuildUUID == "" {
                raw := new([16]byte)
                if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil {
        if BuildUUID == "" {
                raw := new([16]byte)
                if _, err = io.ReadFull(rand.Reader, raw[:]); err != nil {


@@ -189,6 +234,7 @@ func main() {
                        tgts = []string{"all"}
                }
        }
                        tgts = []string{"all"}
                }
        }

+

        statusInit()
 
        for i, tgt := range tgts {
        statusInit()
 
        for i, tgt := range tgts {


@@ -264,17 +310,19 @@ CmdSwitch:
                        log.Fatalln("single target expected")
                }
                var fdTmp *os.File
                        log.Fatalln("single target expected")
                }
                var fdTmp *os.File

-               fdTmp, err = ioutil.TempFile("", "")
+               fdTmp, err = ioutil.TempFile("", "whichdo")

                if err != nil {
                        break
                }
                if err != nil {
                        break
                }

-               os.Remove(fdTmp.Name())
+               if err = os.Remove(fdTmp.Name()); err != nil {
+                       break
+               }

                cwd, tgt := cwdAndTgt(tgts[0])
                doFile, upLevels, err := findDo(fdTmp, cwd, tgt)
                if err != nil {
                        break
                }
                cwd, tgt := cwdAndTgt(tgts[0])
                doFile, upLevels, err := findDo(fdTmp, cwd, tgt)
                if err != nil {
                        break
                }

-               _, err = fdTmp.Seek(0, 0)
+               _, err = fdTmp.Seek(0, os.SEEK_SET)

                if err != nil {
                        break
                }
                if err != nil {
                        break
                }

diff --git a/ood.go b/ood.go
index 0552e2016112a46b037976dbfc88f26adf677b2a..cad1ce38f5f23f6beb1124749774a7d4111303ec 100644 (file)
--- a/ood.go
+++ b/ood.go
@@ -26,6 +26,8 @@ import (
        "path"
        "path/filepath"
        "strings"
        "path"
        "path/filepath"
        "strings"

+
+       "golang.org/x/sys/unix"

 )
 
 const (
 )
 
 const (


@@ -33,6 +35,15 @@ const (
        DepTypeIfchange = "ifchange"
        DepTypeAlways   = "always"
        DepTypeStamp    = "stamp"
        DepTypeIfchange = "ifchange"
        DepTypeAlways   = "always"
        DepTypeStamp    = "stamp"

+
+       EnvOODTgtsFd     = "REDO_OOD_TGTS_FD"
+       EnvOODTgtsLockFd = "REDO_OOD_TGTS_LOCK_FD"
+)
+
+var (
+       OODTgts       map[string]struct{}
+       FdOODTgts     *os.File
+       FdOODTgtsLock *os.File

 )
 
 type TgtErr struct {
 )
 
 type TgtErr struct {


@@ -170,7 +181,7 @@ func isOOD(cwd, tgtOrig string, level int, seen map[string]struct{}) (bool, erro
                        continue
                }
 
                        continue
                }
 

-               depOod, err := isOOD(cwd, dep, level+1, seen)
+               depOod, err := isOODWithTrace(cwd, dep, level+1, seen)

                if err != nil {
                        return ood, TgtErr{tgtOrig, err}
                }
                if err != nil {
                        return ood, TgtErr{tgtOrig, err}
                }


@@ -186,3 +197,49 @@ Done:
        trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood)
        return ood, nil
 }
        trace(CDebug, "ood: %s%s: %v", indent, tgtOrig, ood)
        return ood, nil
 }

+
+func isOODWithTrace(
+       cwd, tgtOrig string,
+       level int,
+       seen map[string]struct{},
+) (bool, error) {
+       p, err := filepath.Abs(path.Join(cwd, tgtOrig))
+       if err != nil {
+               panic(err)
+       }
+       _, ood := OODTgts[p]
+       if ood {
+               trace(
+                       CDebug,
+                       "ood: %s%s true, external decision",
+                       strings.Repeat(". ", level), tgtOrig,
+               )
+               goto RecordOODTgt
+       }
+       ood, err = isOOD(cwd, tgtOrig, level, seen)
+       if !ood {
+               return ood, err
+       }
+RecordOODTgt:
+       if err = unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil {
+               panic(err)
+       }
+       if _, err = FdOODTgts.Seek(0, os.SEEK_END); err != nil {
+               panic(err)
+       }
+       if _, err := FdOODTgts.WriteString(p + "\x00"); err != nil {
+               panic(err)
+       }
+       unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN)
+       return true, nil
+}
+
+func oodTgtsClear() {
+       if err := unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_EX); err != nil {
+               panic(err)
+       }
+       if err := FdOODTgts.Truncate(0); err != nil {
+               panic(err)
+       }
+       unix.Flock(int(FdOODTgtsLock.Fd()), unix.LOCK_UN)
+}

diff --git a/run.go b/run.go
index af17eaa415a66df05439c3919a5ccac79ee6147a..af71b3b8b3b4c4b08d39082e295d2fa48316ed62 100644 (file)
--- a/run.go
+++ b/run.go
@@ -345,6 +345,13 @@ func runScript(tgtOrig string, errs chan error, traced bool) error {
        ))
 
        fdNum := 0
        ))
 
        fdNum := 0

+       cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts)
+       cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum))
+       fdNum++
+       cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock)
+       cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum))
+       fdNum++
+

        if FdStatus == nil {
                cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd))
        } else {
        if FdStatus == nil {
                cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd))
        } else {

diff --git a/usage.go b/usage.go
index affd4540c56254502d11290fbb039fa514e65f99..a8bb28b8191849c4289539b7f7cab38351eebbcb 100644 (file)
--- a/usage.go
+++ b/usage.go
@@ -26,7 +26,7 @@ import (
 )
 
 const (
 )
 
 const (

-       Version  = "1.2.0"
+       Version  = "1.3.0"

        Warranty = `This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, version 3 of the License.
        Warranty = `This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, version 3 of the License.