Storage optimisations for the same often used data

[goredo.git] / dep.go

diff --git a/dep.go b/dep.go
index 6644ac54695674cc8c8ad539a1e17ecadc37f7b5..94daa2a7a8be11629ea33715741df2d648a4b6df 100644 (file)
--- a/dep.go
+++ b/dep.go
@@ -38,6 +38,8 @@ var (
        DepCwd    string
 
        ErrBadRecFormat = errors.New("invalid format of .rec")
+       InodeCache      = make(map[string][]*Inode)
+       HashCache       = make(map[string][]string)
 )
 
 func recfileWrite(fdDep io.StringWriter, fields ...recfile.Field) error {
@@ -70,25 +72,25 @@ func stamp(fdDep, src *os.File) error {
        if err != nil {
                return err
        }
-       tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hex.EncodeToString(hsh))
+       tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hex.EncodeToString([]byte(hsh)))
        return recfileWrite(
                fdDep,
                recfile.Field{Name: "Type", Value: DepTypeStamp},
-               recfile.Field{Name: "Hash", Value: hex.EncodeToString(hsh)},
+               recfile.Field{Name: "Hash", Value: hex.EncodeToString([]byte(hsh))},
        )
 }
 
-func fileHash(fd *os.File) ([]byte, error) {
+func fileHash(fd *os.File) (string, error) {
        h := blake3.New(32, nil)
        if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
-               return nil, err
+               return "", err
        }
-       return h.Sum(nil), nil
+       return string(h.Sum(nil)), nil
 }
 
-func depWrite(fdDep *os.File, cwd, tgt string, hsh []byte) error {
+func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh string) error {
        tracef(CDebug, "ifchange: %s <- %s", fdDep.Name(), tgt)
-       fd, err := os.Open(path.Join(cwd, tgt))
+       fd, err := os.Open(tgt.a)
        if err != nil {
                return ErrLine(err)
        }
@@ -100,7 +102,7 @@ func depWrite(fdDep *os.File, cwd, tgt string, hsh []byte) error {
        if isDir {
                return nil
        }
-       if hsh == nil {
+       if hsh == "" {
                hsh, err = fileHash(fd)
                if err != nil {
                        return ErrLine(err)
@@ -108,30 +110,30 @@ func depWrite(fdDep *os.File, cwd, tgt string, hsh []byte) error {
        }
        fields := []recfile.Field{
                {Name: "Type", Value: DepTypeIfchange},
-               {Name: "Target", Value: tgt},
-               {Name: "Hash", Value: hex.EncodeToString(hsh)},
+               {Name: "Target", Value: tgt.RelTo(cwd)},
+               {Name: "Hash", Value: hex.EncodeToString([]byte(hsh))},
        }
        fields = append(fields, inode.RecfileFields()...)
        return recfileWrite(fdDep, fields...)
 }
 
-func depsWrite(fdDep *os.File, tgts []string) error {
+func depsWrite(fdDep *os.File, tgts []*Tgt) error {
        if fdDep == nil {
                tracef(CDebug, "no opened fdDep: %s", tgts)
                return nil
        }
        var err error
+       var cwd string
        for _, tgt := range tgts {
-               tgtAbs := mustAbs(tgt)
-               cwd := Cwd
+               cwd = Cwd
                if DepCwd != "" && Cwd != DepCwd {
                        cwd = DepCwd
                }
                tgtDir := path.Join(cwd, DirPrefix)
-               tgtRel := mustRel(tgtDir, tgtAbs)
-               if _, errStat := os.Stat(tgt); errStat == nil {
-                       err = ErrLine(depWrite(fdDep, tgtDir, tgtRel, nil))
+               if _, errStat := os.Stat(tgt.a); errStat == nil {
+                       err = ErrLine(depWrite(fdDep, tgtDir, tgt, ""))
                } else {
+                       tgtRel := tgt.RelTo(tgtDir)
                        tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDep.Name(), tgtRel)
                        fields := []recfile.Field{
                                {Name: "Type", Value: DepTypeIfchange},
@@ -149,16 +151,16 @@ func depsWrite(fdDep *os.File, tgts []string) error {
 }
 
 type DepInfoIfchange struct {
-       tgt   string
+       tgt   *Tgt
        inode *Inode
-       hash  []byte
+       hash  string
 }
 
 type DepInfo struct {
        build     string
        always    bool
-       stamp     []byte
-       ifcreates []string
+       stamp     string
+       ifcreates []*Tgt
        ifchanges []DepInfoIfchange
 }
 
@@ -172,8 +174,8 @@ func mustHexDecode(s string) []byte {
 
 var missingBuild = errors.New(".rec missing Build:")
 
-func depRead(pth string) (*DepInfo, error) {
-       data, err := os.ReadFile(pth)
+func depRead(tgt *Tgt) (*DepInfo, error) {
+       data, err := os.ReadFile(tgt.Dep())
        if err != nil {
                return nil, err
        }
@@ -204,10 +206,11 @@ func depRead(pth string) (*DepInfo, error) {
                        if dep == "" {
                                return nil, ErrBadRecFormat
                        }
-                       depInfo.ifcreates = append(depInfo.ifcreates, dep)
+                       depInfo.ifcreates = append(depInfo.ifcreates,
+                               NewTgt(path.Join(tgt.h, dep)))
                case DepTypeIfchange:
-                       dep := m["Target"]
-                       if dep == "" {
+                       depRaw := m["Target"]
+                       if depRaw == "" {
                                return nil, ErrBadRecFormat
                        }
                        inode, err := inodeFromRec(m)
@@ -215,7 +218,33 @@ func depRead(pth string) (*DepInfo, error) {
                                log.Print(err)
                                return nil, ErrBadRecFormat
                        }
-                       hsh := mustHexDecode(m["Hash"])
+                       dep := NewTgt(path.Join(tgt.h, depRaw))
+
+                       cachedFound := false
+                       for _, cachedInode := range InodeCache[dep.a] {
+                               if inode.Equals(cachedInode) {
+                                       inode = cachedInode
+                                       cachedFound = true
+                                       break
+                               }
+                       }
+                       if InodeCache != nil && !cachedFound {
+                               InodeCache[dep.a] = append(InodeCache[dep.a], inode)
+                       }
+
+                       hsh := string(mustHexDecode(m["Hash"]))
+                       cachedFound = false
+                       for _, cachedHash := range HashCache[dep.a] {
+                               if hsh == cachedHash {
+                                       hsh = cachedHash
+                                       cachedFound = true
+                                       break
+                               }
+                       }
+                       if HashCache != nil && !cachedFound {
+                               HashCache[dep.a] = append(HashCache[dep.a], hsh)
+                       }
+
                        depInfo.ifchanges = append(depInfo.ifchanges, DepInfoIfchange{
                                tgt: dep, inode: inode, hash: hsh,
                        })
@@ -224,7 +253,7 @@ func depRead(pth string) (*DepInfo, error) {
                        if hsh == "" {
                                return nil, ErrBadRecFormat
                        }
-                       depInfo.stamp = mustHexDecode(hsh)
+                       depInfo.stamp = string(mustHexDecode(hsh))
                default:
                        return nil, ErrBadRecFormat
                }
@@ -232,6 +261,29 @@ func depRead(pth string) (*DepInfo, error) {
        return &depInfo, nil
 }
 
+func depReadOnlyIfchanges(pth string) (ifchanges []string, err error) {
+       data, err := os.ReadFile(pth)
+       if err != nil {
+               return
+       }
+       r := recfile.NewReader(bytes.NewReader(data))
+       var m map[string]string
+       for {
+               m, err = r.NextMap()
+               if err != nil {
+                       if errors.Is(err, io.EOF) {
+                               err = nil
+                               break
+                       }
+                       return
+               }
+               if m["Type"] == DepTypeIfchange {
+                       ifchanges = append(ifchanges, m["Target"])
+               }
+       }
+       return
+}
+
 func depReadBuild(pth string) (string, error) {
        fd, err := os.Open(pth)
        if err != nil {