-/*
-goredo -- djb's redo implementation on pure Go
-Copyright (C) 2020-2021 Sergey Matveev <stargrave@stargrave.org>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
+// goredo -- djb's redo implementation on pure Go
+// Copyright (C) 2020-2024 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Dependencies saver
import (
"bufio"
- "encoding/hex"
+ "bytes"
+ "encoding/binary"
"errors"
"io"
"os"
"path"
- "path/filepath"
- "go.cypherpunks.ru/recfile"
+ "github.com/google/uuid"
"lukechampine.com/blake3"
)
+const (
+ BinMagic = "GOREDO"
+ BinVersionV1 = 0x01
+ UUIDLen = 16
+
+ DepTypeIfcreate = 0x01
+ DepTypeIfchange = 0x02
+ DepTypeAlways = 0x03
+ DepTypeStamp = 0x04
+ DepTypeIfchangeNonex = 0x05
+)
+
var (
DirPrefix string
DepCwd string
- ErrBadRecFormat = errors.New("invalid format of .rec")
+ IfchangeCache = make(map[string][]*Ifchange)
+ DepCache = make(map[string]*Dep)
+
+ NullUUID uuid.UUID
)
-func recfileWrite(fdDep io.StringWriter, fields ...recfile.Field) error {
- w := recfile.NewWriter(fdDep)
- if _, err := w.RecordStart(); err != nil {
- return err
- }
- if _, err := w.WriteFields(fields...); err != nil {
- return err
+func chunkWrite(in []byte) (out []byte) {
+ l := len(in) + 2
+ if l > 1<<16 {
+ panic("too long")
}
- return nil
+ out = make([]byte, l)
+ binary.BigEndian.PutUint16(out[:2], uint16(l))
+ copy(out[2:], in)
+ return
}
-func ifcreate(fdDep *os.File, tgt string) error {
- tracef(CDebug, "ifcreate: %s <- %s", fdDep.Name(), tgt)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: DepTypeIfcreate},
- recfile.Field{Name: "Target", Value: tgt},
- )
+type Ifchange struct {
+ tgt *Tgt
+ meta [InodeLen + HashLen]byte
}
-func always(fdDep *os.File) error {
- tracef(CDebug, "always: %s", fdDep.Name())
- return recfileWrite(fdDep, recfile.Field{Name: "Type", Value: DepTypeAlways})
+func (ifchange *Ifchange) Inode() *Inode {
+ inode := Inode(ifchange.meta[:InodeLen])
+ return &inode
}
-func stamp(fdDep, src *os.File) error {
- var hsh string
- hsh, err := fileHash(src)
- if err != nil {
- return err
- }
- tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hsh)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: DepTypeStamp},
- recfile.Field{Name: "Hash", Value: hsh},
- )
+func (ifchange *Ifchange) Hash() Hash {
+ return Hash(ifchange.meta[InodeLen:])
}
-func fileHash(fd *os.File) (string, error) {
- h := blake3.New(32, nil)
+type Dep struct {
+ build uuid.UUID
+ always bool
+ stamp Hash
+ ifcreates []*Tgt
+ ifchanges []*Ifchange
+}
+
+func ifcreate(w io.Writer, fdDepName string, tgt string) (err error) {
+ tracef(CDebug, "ifcreate: %s <- %s", fdDepName, tgt)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeIfcreate}, []byte(tgt)...,
+ ))))
+ return
+}
+
+func always(w io.Writer, fdDepName string) (err error) {
+ tracef(CDebug, "always: %s", fdDepName)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(
+ []byte{DepTypeAlways},
+ )))
+ return
+}
+
+func stamp(w io.Writer, fdDepName string, hsh Hash) (err error) {
+ tracef(CDebug, "stamp: %s <- %s", fdDepName, hsh)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeStamp}, []byte(hsh)...,
+ ))))
+ return
+}
+
+func fileHash(fd io.Reader) (Hash, error) {
+ h := blake3.New(HashLen, nil)
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
return "", err
}
- return hex.EncodeToString(h.Sum(nil)), nil
+ return Hash(h.Sum(nil)), nil
}
-func depWrite(fdDep *os.File, cwd, tgt string) error {
- tracef(CDebug, "ifchange: %s <- %s", fdDep.Name(), tgt)
- fd, err := os.Open(path.Join(cwd, tgt))
+func depWrite(w io.Writer, fdDepName, cwd string, tgt *Tgt, hsh Hash) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s", fdDepName, tgt)
+ fd, err := os.Open(tgt.a)
if err != nil {
- return err
+ return ErrLine(err)
}
defer fd.Close()
- fi, err := fd.Stat()
+ inode, isDir, err := inodeFromFileByFd(fd)
if err != nil {
- return err
+ return ErrLine(err)
}
- if fi.IsDir() {
+ if isDir {
return nil
}
- inode, err := inodeFromFile(fd)
- if err != nil {
- return err
- }
- hsh, err := fileHash(fd)
- if err != nil {
- return err
- }
- fields := []recfile.Field{
- {Name: "Type", Value: DepTypeIfchange},
- {Name: "Target", Value: tgt},
- {Name: "Hash", Value: hsh},
+ if hsh == "" {
+ hsh, err = fileHash(fd)
+ if err != nil {
+ return ErrLine(err)
+ }
}
- fields = append(fields, inode.RecfileFields()...)
- return recfileWrite(fdDep, fields...)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchange},
+ inode[:],
+ []byte(hsh),
+ []byte(tgt.RelTo(cwd)),
+ }, nil))))
+ return
+}
+
+func depWriteNonex(w io.Writer, fdDepName, tgtRel string) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDepName, tgtRel)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchangeNonex},
+ []byte(tgtRel),
+ }, nil))))
+ return
}
-func depsWrite(fdDep *os.File, tgts []string) error {
+func depsWrite(fdDep *os.File, tgts []*Tgt) error {
if fdDep == nil {
tracef(CDebug, "no opened fdDep: %s", tgts)
return nil
}
+ var err error
+ var cwd string
+ fdDepW := bufio.NewWriter(fdDep)
for _, tgt := range tgts {
- tgtAbs, err := filepath.Abs(tgt)
- if err != nil {
- panic(err)
- }
- cwd := Cwd
+ cwd = Cwd
if DepCwd != "" && Cwd != DepCwd {
cwd = DepCwd
}
tgtDir := path.Join(cwd, DirPrefix)
- tgtRel, err := filepath.Rel(tgtDir, tgtAbs)
- if err != nil {
- panic(err)
- }
- if _, errStat := os.Stat(tgt); errStat == nil {
- err = depWrite(fdDep, tgtDir, tgtRel)
+ if _, errStat := os.Stat(tgt.a); errStat == nil {
+ err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtDir, tgt, ""))
} else {
- tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDep.Name(), tgtRel)
- fields := []recfile.Field{
- {Name: "Type", Value: DepTypeIfchange},
- {Name: "Target", Value: tgtRel},
- }
- inodeDummy := Inode{}
- fields = append(fields, inodeDummy.RecfileFields()...)
- err = recfileWrite(fdDep, fields...)
+ tgtRel := tgt.RelTo(tgtDir)
+ err = ErrLine(depWriteNonex(fdDepW, fdDep.Name(), tgtRel))
}
if err != nil {
return err
}
}
- return nil
+ return fdDepW.Flush()
}
-type DepInfo struct {
- build string
- always bool
- stamp string
- ifcreates []string
- ifchanges []map[string]string
+func depHeadParse(data []byte) (build uuid.UUID, tail []byte, err error) {
+ if len(data) < len(BinMagic)+1+UUIDLen {
+ err = errors.New("too short header")
+ return
+ }
+ if !bytes.Equal(data[:len(BinMagic)], []byte(BinMagic)) {
+ err = errors.New("bad magic")
+ return
+ }
+ data = data[len(BinMagic):]
+ switch data[0] {
+ case BinVersionV1:
+ default:
+ err = errors.New("unknown version")
+ return
+ }
+ build = uuid.Must(uuid.FromBytes(data[1 : 1+UUIDLen]))
+ tail = data[1+UUIDLen:]
+ return
+}
+
+func chunkRead(data []byte) (typ byte, chunk []byte, tail []byte, err error) {
+ if len(data) < 2 {
+ err = errors.New("no length")
+ return
+ }
+ l := binary.BigEndian.Uint16(data[:2])
+ if l == 0 {
+ err = errors.New("zero length chunk")
+ return
+ }
+ if len(data) < int(l) {
+ err = errors.New("not enough data")
+ return
+ }
+ typ, chunk, tail = data[2], data[3:l], data[l:]
+ return
}
-func depRead(fdDep io.Reader) (*DepInfo, error) {
- r := recfile.NewReader(fdDep)
- m, err := r.NextMap()
+func depBinIfchangeParse(tgt *Tgt, chunk []byte) (*Ifchange, string, error) {
+ if len(chunk) < InodeLen+HashLen+1 {
+ return nil, "", errors.New("too short \"ifchange\" format")
+ }
+
+ tgtH, _ := pathSplit(tgt.a)
+ name := string(chunk[InodeLen+HashLen:])
+ ifchange := &Ifchange{
+ tgt: NewTgt(path.Join(tgtH, name)),
+ meta: ([InodeLen + HashLen]byte)(chunk),
+ }
+ for _, cached := range IfchangeCache[ifchange.tgt.rel] {
+ if ifchange.meta == cached.meta {
+ return cached, name, nil
+ }
+ }
+ if IfchangeCache != nil {
+ IfchangeCache[ifchange.tgt.rel] = append(IfchangeCache[ifchange.tgt.rel], ifchange)
+ }
+ return ifchange, name, nil
+}
+
+func depParse(tgt *Tgt, data []byte) (*Dep, error) {
+ build, data, err := depHeadParse(data)
if err != nil {
return nil, err
}
- depInfo := DepInfo{}
- b := m["Build"]
- if b == "" {
- return nil, errors.New(".rec missing Build:")
- }
- depInfo.build = b
- for {
- m, err := r.NextMap()
+ dep := Dep{build: build}
+ var typ byte
+ var chunk []byte
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
if err != nil {
- if errors.Is(err, io.EOF) {
- break
- }
- return nil, err
+ return nil, ErrLine(err)
}
- switch m["Type"] {
+ switch typ {
case DepTypeAlways:
- depInfo.always = true
+ if len(chunk) != 0 {
+ return nil, ErrLine(errors.New("bad \"always\" format"))
+ }
+ dep.always = true
+ case DepTypeStamp:
+ if len(chunk) != HashLen {
+ return nil, ErrLine(errors.New("bad \"stamp\" format"))
+ }
+ dep.stamp = Hash(chunk)
case DepTypeIfcreate:
- dep := m["Target"]
- if dep == "" {
- return nil, ErrBadRecFormat
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifcreate\" format"))
}
- depInfo.ifcreates = append(depInfo.ifcreates, dep)
+ tgtH, _ := pathSplit(tgt.a)
+ dep.ifcreates = append(dep.ifcreates, NewTgt(path.Join(tgtH, string(chunk))))
case DepTypeIfchange:
- delete(m, "Type")
- depInfo.ifchanges = append(depInfo.ifchanges, m)
- case DepTypeStamp:
- hsh := m["Hash"]
- if hsh == "" {
- return nil, ErrBadRecFormat
+ ifchange, _, err := depBinIfchangeParse(tgt, chunk)
+ if err != nil {
+ return nil, ErrLine(err)
}
- depInfo.stamp = hsh
+ dep.ifchanges = append(dep.ifchanges, ifchange)
+ case DepTypeIfchangeNonex:
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifchange\" format"))
+ }
+ dep.ifchanges = append(dep.ifchanges, &Ifchange{tgt: NewTgt(string(chunk))})
default:
- return nil, ErrBadRecFormat
+ return nil, ErrLine(errors.New("unknown type"))
}
}
- return &depInfo, nil
+ return &dep, nil
+}
+
+func depRead(tgt *Tgt) (*Dep, error) {
+ data, err := os.ReadFile(tgt.dep)
+ if err != nil {
+ return nil, err
+ }
+ return depParse(tgt, data)
+}
+
+func depReadOnlyIfchanges(pth string) (ifchanges []string, err error) {
+ data, err := os.ReadFile(pth)
+ if err != nil {
+ return
+ }
+ _, data, err = depHeadParse(data)
+ if err != nil {
+ return nil, err
+ }
+ var typ byte
+ var chunk []byte
+ var tgt string
+ tgtDummy := NewTgt("")
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ switch typ {
+ case DepTypeIfchange:
+ _, tgt, err = depBinIfchangeParse(tgtDummy, chunk)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ ifchanges = append(ifchanges, tgt)
+ case DepTypeIfchangeNonex:
+ ifchanges = append(ifchanges, string(chunk))
+ }
+ }
+ return
+}
+
+func depBuildRead(pth string) (uuid.UUID, error) {
+ fd, err := os.Open(pth)
+ if err != nil {
+ return NullUUID, err
+ }
+ data := make([]byte, len(BinMagic)+1+UUIDLen)
+ _, err = io.ReadFull(fd, data)
+ fd.Close()
+ if err != nil {
+ return NullUUID, err
+ }
+ build, _, err := depHeadParse(data)
+ return build, err
+}
+
+func depBuildWrite(w io.Writer, build uuid.UUID) (err error) {
+ _, err = io.Copy(w, bytes.NewBuffer(bytes.Join([][]byte{
+ []byte(BinMagic),
+ {BinVersionV1},
+ build[:],
+ }, nil)))
+ return
}