Storage optimisations for the same often used data

[goredo.git] / dep.go

diff --git a/dep.go b/dep.go
index e587e6aa88ced9193b503b94ed2302fb459fba46..94daa2a7a8be11629ea33715741df2d648a4b6df 100644 (file)
--- a/dep.go
+++ b/dep.go
@@ -38,6 +38,8 @@ var (
        DepCwd    string
 
        ErrBadRecFormat = errors.New("invalid format of .rec")
+       InodeCache      = make(map[string][]*Inode)
+       HashCache       = make(map[string][]string)
 )
 
 func recfileWrite(fdDep io.StringWriter, fields ...recfile.Field) error {
@@ -70,23 +72,23 @@ func stamp(fdDep, src *os.File) error {
        if err != nil {
                return err
        }
-       tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hex.EncodeToString(hsh))
+       tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hex.EncodeToString([]byte(hsh)))
        return recfileWrite(
                fdDep,
                recfile.Field{Name: "Type", Value: DepTypeStamp},
-               recfile.Field{Name: "Hash", Value: hex.EncodeToString(hsh)},
+               recfile.Field{Name: "Hash", Value: hex.EncodeToString([]byte(hsh))},
        )
 }
 
-func fileHash(fd *os.File) ([]byte, error) {
+func fileHash(fd *os.File) (string, error) {
        h := blake3.New(32, nil)
        if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
-               return nil, err
+               return "", err
        }
-       return h.Sum(nil), nil
+       return string(h.Sum(nil)), nil
 }
 
-func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh []byte) error {
+func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh string) error {
        tracef(CDebug, "ifchange: %s <- %s", fdDep.Name(), tgt)
        fd, err := os.Open(tgt.a)
        if err != nil {
@@ -100,7 +102,7 @@ func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh []byte) error {
        if isDir {
                return nil
        }
-       if hsh == nil {
+       if hsh == "" {
                hsh, err = fileHash(fd)
                if err != nil {
                        return ErrLine(err)
@@ -109,7 +111,7 @@ func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh []byte) error {
        fields := []recfile.Field{
                {Name: "Type", Value: DepTypeIfchange},
                {Name: "Target", Value: tgt.RelTo(cwd)},
-               {Name: "Hash", Value: hex.EncodeToString(hsh)},
+               {Name: "Hash", Value: hex.EncodeToString([]byte(hsh))},
        }
        fields = append(fields, inode.RecfileFields()...)
        return recfileWrite(fdDep, fields...)
@@ -129,7 +131,7 @@ func depsWrite(fdDep *os.File, tgts []*Tgt) error {
                }
                tgtDir := path.Join(cwd, DirPrefix)
                if _, errStat := os.Stat(tgt.a); errStat == nil {
-                       err = ErrLine(depWrite(fdDep, tgtDir, tgt, nil))
+                       err = ErrLine(depWrite(fdDep, tgtDir, tgt, ""))
                } else {
                        tgtRel := tgt.RelTo(tgtDir)
                        tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDep.Name(), tgtRel)
@@ -151,13 +153,13 @@ func depsWrite(fdDep *os.File, tgts []*Tgt) error {
 type DepInfoIfchange struct {
        tgt   *Tgt
        inode *Inode
-       hash  []byte
+       hash  string
 }
 
 type DepInfo struct {
        build     string
        always    bool
-       stamp     []byte
+       stamp     string
        ifcreates []*Tgt
        ifchanges []DepInfoIfchange
 }
@@ -207,8 +209,8 @@ func depRead(tgt *Tgt) (*DepInfo, error) {
                        depInfo.ifcreates = append(depInfo.ifcreates,
                                NewTgt(path.Join(tgt.h, dep)))
                case DepTypeIfchange:
-                       dep := m["Target"]
-                       if dep == "" {
+                       depRaw := m["Target"]
+                       if depRaw == "" {
                                return nil, ErrBadRecFormat
                        }
                        inode, err := inodeFromRec(m)
@@ -216,16 +218,42 @@ func depRead(tgt *Tgt) (*DepInfo, error) {
                                log.Print(err)
                                return nil, ErrBadRecFormat
                        }
-                       hsh := mustHexDecode(m["Hash"])
+                       dep := NewTgt(path.Join(tgt.h, depRaw))
+
+                       cachedFound := false
+                       for _, cachedInode := range InodeCache[dep.a] {
+                               if inode.Equals(cachedInode) {
+                                       inode = cachedInode
+                                       cachedFound = true
+                                       break
+                               }
+                       }
+                       if InodeCache != nil && !cachedFound {
+                               InodeCache[dep.a] = append(InodeCache[dep.a], inode)
+                       }
+
+                       hsh := string(mustHexDecode(m["Hash"]))
+                       cachedFound = false
+                       for _, cachedHash := range HashCache[dep.a] {
+                               if hsh == cachedHash {
+                                       hsh = cachedHash
+                                       cachedFound = true
+                                       break
+                               }
+                       }
+                       if HashCache != nil && !cachedFound {
+                               HashCache[dep.a] = append(HashCache[dep.a], hsh)
+                       }
+
                        depInfo.ifchanges = append(depInfo.ifchanges, DepInfoIfchange{
-                               tgt: NewTgt(path.Join(tgt.h, dep)), inode: inode, hash: hsh,
+                               tgt: dep, inode: inode, hash: hsh,
                        })
                case DepTypeStamp:
                        hsh := m["Hash"]
                        if hsh == "" {
                                return nil, ErrBadRecFormat
                        }
-                       depInfo.stamp = mustHexDecode(hsh)
+                       depInfo.stamp = string(mustHexDecode(hsh))
                default:
                        return nil, ErrBadRecFormat
                }