import (
"bufio"
"bytes"
- "encoding/hex"
+ "encoding/binary"
"errors"
"io"
- "log"
"os"
"path"
- "go.cypherpunks.ru/recfile"
+ "github.com/google/uuid"
"lukechampine.com/blake3"
)
+const (
+ BinMagic = "GOREDO"
+ BinVersionV1 = 0x01
+ UUIDLen = 16
+
+ DepTypeIfcreate = 0x01
+ DepTypeIfchange = 0x02
+ DepTypeAlways = 0x03
+ DepTypeStamp = 0x04
+ DepTypeIfchangeDummy = 0x05
+)
+
var (
DirPrefix string
DepCwd string
- ErrBadRecFormat = errors.New("invalid format of .rec")
- InodeCache = make(map[string][]*Inode)
- HashCache = make(map[string][]Hash)
+ IfchangeCache = make(map[string][]*Ifchange)
+ DepCache = make(map[string]*Dep)
+
+ NullUUID uuid.UUID
)
-func recfileWrite(fdDep io.StringWriter, fields ...recfile.Field) error {
- w := recfile.NewWriter(fdDep)
- if _, err := w.RecordStart(); err != nil {
- return err
- }
- if _, err := w.WriteFields(fields...); err != nil {
- return err
+func chunkWrite(in []byte) (out []byte) {
+ l := len(in) + 2
+ if l > 1<<16 {
+ panic("too long")
}
- return nil
+ out = make([]byte, l)
+ binary.BigEndian.PutUint16(out[:2], uint16(l))
+ copy(out[2:], in)
+ return
}
-func ifcreate(fdDep *os.File, tgt string) error {
- tracef(CDebug, "ifcreate: %s <- %s", fdDep.Name(), tgt)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: DepTypeIfcreate},
- recfile.Field{Name: "Target", Value: tgt},
- )
+type Ifchange struct {
+ tgt *Tgt
+ meta string
}
-func always(fdDep *os.File) error {
- tracef(CDebug, "always: %s", fdDep.Name())
- return recfileWrite(fdDep, recfile.Field{Name: "Type", Value: DepTypeAlways})
+func (ifchange *Ifchange) Inode() Inode {
+ return Inode(ifchange.meta[:InodeLen])
}
-func stamp(fdDep, src *os.File) error {
- hsh, err := fileHash(src)
- if err != nil {
- return err
- }
- tracef(CDebug, "stamp: %s <- %s", fdDep.Name(), hsh)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: DepTypeStamp},
- recfile.Field{Name: "Hash", Value: hsh.String()},
- )
+func (ifchange *Ifchange) Hash() Hash {
+ return Hash(ifchange.meta[InodeLen:])
+}
+
+type Dep struct {
+ build uuid.UUID
+ always bool
+ stamp Hash
+ ifcreates []*Tgt
+ ifchanges []*Ifchange
+}
+
+func ifcreate(w io.Writer, fdDepName string, tgt string) (err error) {
+ tracef(CDebug, "ifcreate: %s <- %s", fdDepName, tgt)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeIfcreate}, []byte(tgt)...,
+ ))))
+ return
+}
+
+func always(w io.Writer, fdDepName string) (err error) {
+ tracef(CDebug, "always: %s", fdDepName)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(
+ []byte{DepTypeAlways},
+ )))
+ return
+}
+
+func stamp(w io.Writer, fdDepName string, hsh Hash) (err error) {
+ tracef(CDebug, "stamp: %s <- %s", fdDepName, hsh)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeStamp}, []byte(hsh)...,
+ ))))
+ return
}
-func fileHash(fd *os.File) (Hash, error) {
+func fileHash(fd io.Reader) (Hash, error) {
h := blake3.New(HashLen, nil)
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
return "", err
return Hash(h.Sum(nil)), nil
}
-func depWrite(fdDep *os.File, cwd string, tgt *Tgt, hsh Hash) error {
- tracef(CDebug, "ifchange: %s <- %s", fdDep.Name(), tgt)
+func depWrite(w io.Writer, fdDepName, cwd string, tgt *Tgt, hsh Hash) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s", fdDepName, tgt)
fd, err := os.Open(tgt.a)
if err != nil {
return ErrLine(err)
return ErrLine(err)
}
}
- fields := []recfile.Field{
- {Name: "Type", Value: DepTypeIfchange},
- {Name: "Target", Value: tgt.RelTo(cwd)},
- {Name: "Hash", Value: hsh.String()},
- }
- fields = append(fields, inode.RecfileFields()...)
- return recfileWrite(fdDep, fields...)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchange},
+ []byte(inode),
+ []byte(hsh),
+ []byte(tgt.RelTo(cwd)),
+ }, nil))))
+ return
+}
+
+func depWriteDummy(w io.Writer, fdDepName, tgtRel string) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDepName, tgtRel)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchangeDummy},
+ []byte(tgtRel),
+ }, nil))))
+ return
}
func depsWrite(fdDep *os.File, tgts []*Tgt) error {
}
var err error
var cwd string
+ fdDepW := bufio.NewWriter(fdDep)
for _, tgt := range tgts {
cwd = Cwd
if DepCwd != "" && Cwd != DepCwd {
}
tgtDir := path.Join(cwd, DirPrefix)
if _, errStat := os.Stat(tgt.a); errStat == nil {
- err = ErrLine(depWrite(fdDep, tgtDir, tgt, ""))
+ err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtDir, tgt, ""))
} else {
tgtRel := tgt.RelTo(tgtDir)
- tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDep.Name(), tgtRel)
- fields := []recfile.Field{
- {Name: "Type", Value: DepTypeIfchange},
- {Name: "Target", Value: tgtRel},
- }
- inodeDummy := Inode{}
- fields = append(fields, inodeDummy.RecfileFields()...)
- err = ErrLine(recfileWrite(fdDep, fields...))
+ err = ErrLine(depWriteDummy(fdDepW, fdDep.Name(), tgtRel))
}
if err != nil {
return err
}
}
- return nil
+ return fdDepW.Flush()
}
-type DepInfoIfchange struct {
- tgt *Tgt
- inode *Inode
- hash Hash
-}
-
-type DepInfo struct {
- build string
- always bool
- stamp Hash
- ifcreates []*Tgt
- ifchanges []DepInfoIfchange
+func depHeadParse(data []byte) (build uuid.UUID, tail []byte, err error) {
+ if len(data) < len(BinMagic)+1+UUIDLen {
+ err = errors.New("too short header")
+ return
+ }
+ if !bytes.Equal(data[:len(BinMagic)], []byte(BinMagic)) {
+ err = errors.New("bad magic")
+ return
+ }
+ data = data[len(BinMagic):]
+ switch data[0] {
+ case BinVersionV1:
+ default:
+ err = errors.New("unknown version")
+ return
+ }
+ build = uuid.Must(uuid.FromBytes(data[1 : 1+UUIDLen]))
+ tail = data[1+UUIDLen:]
+ return
}
-func mustHashDecode(s string) Hash {
- b, err := hex.DecodeString(s)
- if err != nil {
- log.Fatal(err)
+func chunkRead(data []byte) (typ byte, chunk []byte, tail []byte, err error) {
+ if len(data) < 2 {
+ err = errors.New("no length")
+ return
}
- return Hash(b)
+ l := binary.BigEndian.Uint16(data[:2])
+ if l == 0 {
+ err = errors.New("zero length chunk")
+ return
+ }
+ if len(data) < int(l) {
+ err = errors.New("not enough data")
+ return
+ }
+ typ, chunk, tail = data[2], data[3:l], data[l:]
+ return
}
-var missingBuild = errors.New(".rec missing Build:")
+func depBinIfchangeParse(tgt *Tgt, chunk []byte) (*Ifchange, string, error) {
+ if len(chunk) < InodeLen+HashLen+1 {
+ return nil, "", errors.New("too short \"ifchange\" format")
+ }
+ name := string(chunk[InodeLen+HashLen:])
+ meta := string(chunk[:InodeLen+HashLen])
-func depRead(tgt *Tgt) (*DepInfo, error) {
- data, err := os.ReadFile(tgt.Dep())
- if err != nil {
- return nil, err
+ tgtH, _ := pathSplit(tgt.a)
+ ifchange := &Ifchange{tgt: NewTgt(path.Join(tgtH, name)), meta: meta}
+ cachedFound := false
+ for _, cached := range IfchangeCache[ifchange.tgt.rel] {
+ if ifchange.meta == cached.meta {
+ ifchange = cached
+ cachedFound = true
+ break
+ }
+ }
+ if IfchangeCache != nil && !cachedFound {
+ IfchangeCache[ifchange.tgt.rel] = append(IfchangeCache[ifchange.tgt.rel], ifchange)
}
- r := recfile.NewReader(bytes.NewReader(data))
- m, err := r.NextMap()
+ return ifchange, name, nil
+}
+
+func depParse(tgt *Tgt, data []byte) (*Dep, error) {
+ build, data, err := depHeadParse(data)
if err != nil {
return nil, err
}
- depInfo := DepInfo{}
- b := m["Build"]
- if b == "" {
- return nil, missingBuild
- }
- depInfo.build = b
- for {
- m, err := r.NextMap()
+ dep := Dep{build: build}
+ var typ byte
+ var chunk []byte
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
if err != nil {
- if errors.Is(err, io.EOF) {
- break
- }
- return nil, err
+ return nil, ErrLine(err)
}
- switch m["Type"] {
+ switch typ {
case DepTypeAlways:
- depInfo.always = true
+ if len(chunk) != 0 {
+ return nil, ErrLine(errors.New("bad \"always\" format"))
+ }
+ dep.always = true
+ case DepTypeStamp:
+ if len(chunk) != HashLen {
+ return nil, ErrLine(errors.New("bad \"stamp\" format"))
+ }
+ dep.stamp = Hash(chunk)
case DepTypeIfcreate:
- dep := m["Target"]
- if dep == "" {
- return nil, ErrBadRecFormat
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifcreate\" format"))
}
- depInfo.ifcreates = append(depInfo.ifcreates,
- NewTgt(path.Join(tgt.h, dep)))
+ tgtH, _ := pathSplit(tgt.a)
+ dep.ifcreates = append(dep.ifcreates, NewTgt(path.Join(tgtH, string(chunk))))
case DepTypeIfchange:
- depRaw := m["Target"]
- if depRaw == "" {
- return nil, ErrBadRecFormat
- }
- inode, err := inodeFromRec(m)
+ ifchange, _, err := depBinIfchangeParse(tgt, chunk)
if err != nil {
- log.Print(err)
- return nil, ErrBadRecFormat
- }
- dep := NewTgt(path.Join(tgt.h, depRaw))
-
- cachedFound := false
- for _, cachedInode := range InodeCache[dep.a] {
- if inode.Equals(cachedInode) {
- inode = cachedInode
- cachedFound = true
- break
- }
- }
- if InodeCache != nil && !cachedFound {
- InodeCache[dep.a] = append(InodeCache[dep.a], inode)
- }
-
- hsh := mustHashDecode(m["Hash"])
- cachedFound = false
- for _, cachedHash := range HashCache[dep.a] {
- if hsh == cachedHash {
- hsh = cachedHash
- cachedFound = true
- break
- }
+ return nil, ErrLine(err)
}
- if HashCache != nil && !cachedFound {
- HashCache[dep.a] = append(HashCache[dep.a], hsh)
+ dep.ifchanges = append(dep.ifchanges, ifchange)
+ case DepTypeIfchangeDummy:
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifchange\" format"))
}
-
- depInfo.ifchanges = append(depInfo.ifchanges, DepInfoIfchange{
- tgt: dep, inode: inode, hash: hsh,
- })
- case DepTypeStamp:
- hsh := m["Hash"]
- if hsh == "" {
- return nil, ErrBadRecFormat
- }
- depInfo.stamp = mustHashDecode(hsh)
+ dep.ifchanges = append(dep.ifchanges, &Ifchange{tgt: NewTgt(string(chunk))})
default:
- return nil, ErrBadRecFormat
+ return nil, ErrLine(errors.New("unknown type"))
}
}
- return &depInfo, nil
+ return &dep, nil
+}
+
+func depRead(tgt *Tgt) (*Dep, error) {
+ data, err := os.ReadFile(tgt.dep)
+ if err != nil {
+ return nil, err
+ }
+ return depParse(tgt, data)
}
func depReadOnlyIfchanges(pth string) (ifchanges []string, err error) {
if err != nil {
return
}
- r := recfile.NewReader(bytes.NewReader(data))
- var m map[string]string
- for {
- m, err = r.NextMap()
+ _, data, err = depHeadParse(data)
+ if err != nil {
+ return nil, err
+ }
+ var typ byte
+ var chunk []byte
+ var tgt string
+ tgtDummy := NewTgt("")
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
if err != nil {
- if errors.Is(err, io.EOF) {
- err = nil
- break
- }
- return
+ return nil, ErrLine(err)
}
- if m["Type"] == DepTypeIfchange {
- ifchanges = append(ifchanges, m["Target"])
+ switch typ {
+ case DepTypeIfchange:
+ _, tgt, err = depBinIfchangeParse(tgtDummy, chunk)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ ifchanges = append(ifchanges, tgt)
+ case DepTypeIfchangeDummy:
+ ifchanges = append(ifchanges, string(chunk))
}
}
return
}
-func depReadBuild(pth string) (string, error) {
+func depBuildRead(pth string) (uuid.UUID, error) {
fd, err := os.Open(pth)
if err != nil {
- return "", err
+ return NullUUID, err
}
- r := recfile.NewReader(fd)
- m, err := r.NextMap()
+ data := make([]byte, len(BinMagic)+1+UUIDLen)
+ _, err = io.ReadFull(fd, data)
fd.Close()
if err != nil {
- return "", err
- }
- build := m["Build"]
- if build == "" {
- err = missingBuild
+ return NullUUID, err
}
+ build, _, err := depHeadParse(data)
return build, err
}
+
+func depBuildWrite(w io.Writer, build uuid.UUID) (err error) {
+ _, err = io.Copy(w, bytes.NewBuffer(bytes.Join([][]byte{
+ []byte(BinMagic),
+ {BinVersionV1},
+ build[:],
+ }, nil)))
+ return
+}