-/*
-goredo -- redo implementation on pure Go
-Copyright (C) 2020 Sergey Matveev <stargrave@stargrave.org>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
+// goredo -- djb's redo implementation on pure Go
+// Copyright (C) 2020-2024 Sergey Matveev <stargrave@stargrave.org>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Dependencies saver
import (
"bufio"
- "encoding/hex"
- "fmt"
+ "bytes"
+ "encoding/binary"
+ "errors"
"io"
"os"
"path"
- "strings"
- "syscall"
- "go.cypherpunks.ru/recfile"
- "golang.org/x/crypto/blake2b"
+ "github.com/google/uuid"
+ "lukechampine.com/blake3"
)
-const EnvNoHash = "REDO_NO_HASH"
+const (
+ BinMagic = "GOREDO"
+ BinVersionV1 = 0x01
+ UUIDLen = 16
-var NoHash bool
+ DepTypeIfcreate = 0x01
+ DepTypeIfchange = 0x02
+ DepTypeAlways = 0x03
+ DepTypeStamp = 0x04
+ DepTypeIfchangeNonex = 0x05
+)
-func recfileWrite(fdDep *os.File, fields ...recfile.Field) error {
- w := recfile.NewWriter(fdDep)
- if _, err := w.RecordStart(); err != nil {
- return err
- }
- if _, err := w.WriteFields(fields...); err != nil {
- return err
+var (
+ DirPrefix string
+ DepCwd string
+
+ IfchangeCache = make(map[string][]*Ifchange)
+ DepCache = make(map[string]*Dep)
+
+ NullUUID uuid.UUID
+)
+
+func chunkWrite(in []byte) (out []byte) {
+ l := len(in) + 2
+ if l > 1<<16 {
+ panic("too long")
}
- return nil
+ out = make([]byte, l)
+ binary.BigEndian.PutUint16(out[:2], uint16(l))
+ copy(out[2:], in)
+ return
}
-func ifcreate(fdDep *os.File, tgt string) error {
- trace(CDebug, "ifcreate: %s <- %s", fdDep.Name(), tgt)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: "ifcreate"},
- recfile.Field{Name: "Target", Value: tgt},
- )
+type Ifchange struct {
+ tgt *Tgt
+ meta [InodeLen + HashLen]byte
}
-func always(fdDep *os.File) error {
- trace(CDebug, "always: %s", fdDep.Name())
- return recfileWrite(fdDep, recfile.Field{Name: "Type", Value: "always"})
+func (ifchange *Ifchange) Inode() *Inode {
+ inode := Inode(ifchange.meta[:InodeLen])
+ return &inode
}
-func stamp(fdDep, src *os.File) error {
- var hsh string
- hsh, err := fileHash(os.Stdin)
- if err != nil {
- return err
- }
- trace(CDebug, "stamp: %s <- %s", fdDep.Name(), hsh)
- return recfileWrite(
- fdDep,
- recfile.Field{Name: "Type", Value: "stamp"},
- recfile.Field{Name: "Hash", Value: hsh},
- )
+func (ifchange *Ifchange) Hash() Hash {
+ return Hash(ifchange.meta[InodeLen:])
}
-func fileCtime(fd *os.File) (string, error) {
- fi, err := fd.Stat()
- if err != nil {
- return "", err
- }
- stat := fi.Sys().(*syscall.Stat_t)
- sec, nsec := stat.Ctimespec.Unix()
- return fmt.Sprintf("%d.%d", sec, nsec), nil
+type Dep struct {
+ build uuid.UUID
+ always bool
+ stamp Hash
+ ifcreates []*Tgt
+ ifchanges []*Ifchange
}
-func fileHash(fd *os.File) (string, error) {
- h, err := blake2b.New256(nil)
- if err != nil {
- panic(err)
- }
- if _, err = io.Copy(h, bufio.NewReader(fd)); err != nil {
+func ifcreate(w io.Writer, fdDepName string, tgt string) (err error) {
+ tracef(CDebug, "ifcreate: %s <- %s", fdDepName, tgt)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeIfcreate}, []byte(tgt)...,
+ ))))
+ return
+}
+
+func always(w io.Writer, fdDepName string) (err error) {
+ tracef(CDebug, "always: %s", fdDepName)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(
+ []byte{DepTypeAlways},
+ )))
+ return
+}
+
+func stamp(w io.Writer, fdDepName string, hsh Hash) (err error) {
+ tracef(CDebug, "stamp: %s <- %s", fdDepName, hsh)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append(
+ []byte{DepTypeStamp}, []byte(hsh)...,
+ ))))
+ return
+}
+
+func fileHash(fd io.Reader) (Hash, error) {
+ h := blake3.New(HashLen, nil)
+ if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
return "", err
}
- return hex.EncodeToString(h.Sum(nil)), nil
+ return Hash(h.Sum(nil)), nil
}
-func writeDep(fdDep *os.File, cwd, tgt string) error {
- trace(CDebug, "ifchange: %s <- %s", fdDep.Name(), tgt)
- fd, err := os.Open(path.Join(cwd, tgt))
+func depWrite(w io.Writer, fdDepName, cwd string, tgt *Tgt, hsh Hash) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s", fdDepName, tgt)
+ fd, err := os.Open(tgt.a)
if err != nil {
- return err
+ return ErrLine(err)
}
defer fd.Close()
- ts, err := fileCtime(fd)
+ inode, isDir, err := inodeFromFileByFd(fd)
if err != nil {
- return err
+ return ErrLine(err)
}
- fields := []recfile.Field{
- recfile.Field{Name: "Type", Value: "ifchange"},
- recfile.Field{Name: "Target", Value: tgt},
- recfile.Field{Name: "Ctime", Value: ts},
+ if isDir {
+ return nil
}
- var hsh string
- if !NoHash {
+ if hsh == "" {
hsh, err = fileHash(fd)
if err != nil {
- return err
+ return ErrLine(err)
}
- fields = append(fields, recfile.Field{Name: "Hash", Value: hsh})
}
- err = recfileWrite(fdDep, fields...)
- return err
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchange},
+ inode[:],
+ []byte(hsh),
+ []byte(tgt.RelTo(cwd)),
+ }, nil))))
+ return
}
-func writeDeps(fdDep *os.File, tgts []string) error {
+func depWriteNonex(w io.Writer, fdDepName, tgtRel string) (err error) {
+ tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDepName, tgtRel)
+ _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{
+ {DepTypeIfchangeNonex},
+ []byte(tgtRel),
+ }, nil))))
+ return
+}
+
+func depsWrite(fdDep *os.File, tgts []*Tgt) error {
if fdDep == nil {
- trace(CDebug, "no opened fdDep: %s", tgts)
+ tracef(CDebug, "no opened fdDep: %s", tgts)
return nil
}
- ups := []string{}
- upLevels := strings.Count(os.Getenv(EnvDirPrefix), "/")
- for i := 0; i < upLevels; i++ {
- ups = append(ups, "..")
- }
- up := path.Join(ups...)
+ var err error
+ var cwd string
+ fdDepW := bufio.NewWriter(fdDep)
for _, tgt := range tgts {
- if _, err := os.Stat(tgt); err == nil {
- if err = writeDep(fdDep, Cwd, path.Join(up, tgt)); err != nil {
- return err
+ cwd = Cwd
+ if DepCwd != "" && Cwd != DepCwd {
+ cwd = DepCwd
+ }
+ tgtDir := path.Join(cwd, DirPrefix)
+ if _, errStat := os.Stat(tgt.a); errStat == nil {
+ err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtDir, tgt, ""))
+ } else {
+ tgtRel := tgt.RelTo(tgtDir)
+ err = ErrLine(depWriteNonex(fdDepW, fdDep.Name(), tgtRel))
+ }
+ if err != nil {
+ return err
+ }
+ }
+ return fdDepW.Flush()
+}
+
+func depHeadParse(data []byte) (build uuid.UUID, tail []byte, err error) {
+ if len(data) < len(BinMagic)+1+UUIDLen {
+ err = errors.New("too short header")
+ return
+ }
+ if !bytes.Equal(data[:len(BinMagic)], []byte(BinMagic)) {
+ err = errors.New("bad magic")
+ return
+ }
+ data = data[len(BinMagic):]
+ switch data[0] {
+ case BinVersionV1:
+ default:
+ err = errors.New("unknown version")
+ return
+ }
+ build = uuid.Must(uuid.FromBytes(data[1 : 1+UUIDLen]))
+ tail = data[1+UUIDLen:]
+ return
+}
+
+func chunkRead(data []byte) (typ byte, chunk []byte, tail []byte, err error) {
+ if len(data) < 2 {
+ err = errors.New("no length")
+ return
+ }
+ l := binary.BigEndian.Uint16(data[:2])
+ if l == 0 {
+ err = errors.New("zero length chunk")
+ return
+ }
+ if len(data) < int(l) {
+ err = errors.New("not enough data")
+ return
+ }
+ typ, chunk, tail = data[2], data[3:l], data[l:]
+ return
+}
+
+func depBinIfchangeParse(tgt *Tgt, chunk []byte) (*Ifchange, string, error) {
+ if len(chunk) < InodeLen+HashLen+1 {
+ return nil, "", errors.New("too short \"ifchange\" format")
+ }
+
+ tgtH, _ := pathSplit(tgt.a)
+ name := string(chunk[InodeLen+HashLen:])
+ ifchange := &Ifchange{
+ tgt: NewTgt(path.Join(tgtH, name)),
+ meta: ([InodeLen + HashLen]byte)(chunk),
+ }
+ for _, cached := range IfchangeCache[ifchange.tgt.rel] {
+ if ifchange.meta == cached.meta {
+ return cached, name, nil
+ }
+ }
+ if IfchangeCache != nil {
+ IfchangeCache[ifchange.tgt.rel] = append(IfchangeCache[ifchange.tgt.rel], ifchange)
+ }
+ return ifchange, name, nil
+}
+
+func depParse(tgt *Tgt, data []byte) (*Dep, error) {
+ build, data, err := depHeadParse(data)
+ if err != nil {
+ return nil, err
+ }
+ dep := Dep{build: build}
+ var typ byte
+ var chunk []byte
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ switch typ {
+ case DepTypeAlways:
+ if len(chunk) != 0 {
+ return nil, ErrLine(errors.New("bad \"always\" format"))
+ }
+ dep.always = true
+ case DepTypeStamp:
+ if len(chunk) != HashLen {
+ return nil, ErrLine(errors.New("bad \"stamp\" format"))
+ }
+ dep.stamp = Hash(chunk)
+ case DepTypeIfcreate:
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifcreate\" format"))
+ }
+ tgtH, _ := pathSplit(tgt.a)
+ dep.ifcreates = append(dep.ifcreates, NewTgt(path.Join(tgtH, string(chunk))))
+ case DepTypeIfchange:
+ ifchange, _, err := depBinIfchangeParse(tgt, chunk)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ dep.ifchanges = append(dep.ifchanges, ifchange)
+ case DepTypeIfchangeNonex:
+ if len(chunk) < 1 {
+ return nil, ErrLine(errors.New("too short \"ifchange\" format"))
+ }
+ dep.ifchanges = append(dep.ifchanges, &Ifchange{tgt: NewTgt(string(chunk))})
+ default:
+ return nil, ErrLine(errors.New("unknown type"))
+ }
+ }
+ return &dep, nil
+}
+
+func depRead(tgt *Tgt) (*Dep, error) {
+ data, err := os.ReadFile(tgt.dep)
+ if err != nil {
+ return nil, err
+ }
+ return depParse(tgt, data)
+}
+
+func depReadOnlyIfchanges(pth string) (ifchanges []string, err error) {
+ data, err := os.ReadFile(pth)
+ if err != nil {
+ return
+ }
+ _, data, err = depHeadParse(data)
+ if err != nil {
+ return nil, err
+ }
+ var typ byte
+ var chunk []byte
+ var tgt string
+ tgtDummy := NewTgt("")
+ for len(data) > 0 {
+ typ, chunk, data, err = chunkRead(data)
+ if err != nil {
+ return nil, ErrLine(err)
+ }
+ switch typ {
+ case DepTypeIfchange:
+ _, tgt, err = depBinIfchangeParse(tgtDummy, chunk)
+ if err != nil {
+ return nil, ErrLine(err)
}
+ ifchanges = append(ifchanges, tgt)
+ case DepTypeIfchangeNonex:
+ ifchanges = append(ifchanges, string(chunk))
}
}
- return nil
+ return
+}
+
+func depBuildRead(pth string) (uuid.UUID, error) {
+ fd, err := os.Open(pth)
+ if err != nil {
+ return NullUUID, err
+ }
+ data := make([]byte, len(BinMagic)+1+UUIDLen)
+ _, err = io.ReadFull(fd, data)
+ fd.Close()
+ if err != nil {
+ return NullUUID, err
+ }
+ build, _, err := depHeadParse(data)
+ return build, err
+}
+
+func depBuildWrite(w io.Writer, build uuid.UUID) (err error) {
+ _, err = io.Copy(w, bytes.NewBuffer(bytes.Join([][]byte{
+ []byte(BinMagic),
+ {BinVersionV1},
+ build[:],
+ }, nil)))
+ return
}