// goredo -- djb's redo implementation on pure Go // Copyright (C) 2020-2024 Sergey Matveev // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, version 3 of the License. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // Dependencies saver package main import ( "bufio" "bytes" "encoding/binary" "errors" "io" "os" "path" "github.com/google/uuid" "lukechampine.com/blake3" ) const ( BinMagic = "GOREDO" BinVersionV1 = 0x01 UUIDLen = 16 DepTypeIfcreate = 0x01 DepTypeIfchange = 0x02 DepTypeAlways = 0x03 DepTypeStamp = 0x04 DepTypeIfchangeNonex = 0x05 ) var ( DirPrefix string DepCwd string IfchangeCache = make(map[string][]*Ifchange) DepCache = make(map[string]*Dep) NullUUID uuid.UUID ) func chunkWrite(in []byte) (out []byte) { l := len(in) + 2 if l > 1<<16 { panic("too long") } out = make([]byte, l) binary.BigEndian.PutUint16(out[:2], uint16(l)) copy(out[2:], in) return } type Ifchange struct { tgt *Tgt meta [InodeLen + HashLen]byte } func (ifchange *Ifchange) Inode() *Inode { inode := Inode(ifchange.meta[:InodeLen]) return &inode } func (ifchange *Ifchange) Hash() Hash { return Hash(ifchange.meta[InodeLen:]) } type Dep struct { build uuid.UUID always bool stamp Hash ifcreates []*Tgt ifchanges []*Ifchange } func ifcreate(w io.Writer, fdDepName string, tgt string) (err error) { tracef(CDebug, "ifcreate: %s <- %s", fdDepName, tgt) _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append( []byte{DepTypeIfcreate}, []byte(tgt)..., )))) return } func always(w io.Writer, fdDepName string) (err error) { tracef(CDebug, "always: %s", fdDepName) _, err = io.Copy(w, bytes.NewBuffer(chunkWrite( []byte{DepTypeAlways}, ))) return } func stamp(w io.Writer, fdDepName string, hsh Hash) (err error) { tracef(CDebug, "stamp: %s <- %s", fdDepName, hsh) _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(append( []byte{DepTypeStamp}, []byte(hsh)..., )))) return } func fileHash(fd io.Reader) (Hash, error) { h := blake3.New(HashLen, nil) if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { return "", err } return Hash(h.Sum(nil)), nil } func depWrite(w io.Writer, fdDepName, cwd string, tgt *Tgt, hsh Hash) (err error) { tracef(CDebug, "ifchange: %s <- %s", fdDepName, tgt) fd, err := os.Open(tgt.a) if err != nil { return ErrLine(err) } defer fd.Close() inode, isDir, err := inodeFromFileByFd(fd) if err != nil { return ErrLine(err) } if isDir { return nil } if hsh == "" { hsh, err = fileHash(fd) if err != nil { return ErrLine(err) } } _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{ {DepTypeIfchange}, inode[:], []byte(hsh), []byte(tgt.RelTo(cwd)), }, nil)))) return } func depWriteNonex(w io.Writer, fdDepName, tgtRel string) (err error) { tracef(CDebug, "ifchange: %s <- %s (non-existing)", fdDepName, tgtRel) _, err = io.Copy(w, bytes.NewBuffer(chunkWrite(bytes.Join([][]byte{ {DepTypeIfchangeNonex}, []byte(tgtRel), }, nil)))) return } func depsWrite(fdDep *os.File, tgts []*Tgt) error { if fdDep == nil { tracef(CDebug, "no opened fdDep: %s", tgts) return nil } var err error var cwd string fdDepW := bufio.NewWriter(fdDep) for _, tgt := range tgts { cwd = Cwd if DepCwd != "" && Cwd != DepCwd { cwd = DepCwd } tgtDir := path.Join(cwd, DirPrefix) if _, errStat := os.Stat(tgt.a); errStat == nil { err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtDir, tgt, "")) } else { tgtRel := tgt.RelTo(tgtDir) err = ErrLine(depWriteNonex(fdDepW, fdDep.Name(), tgtRel)) } if err != nil { return err } } return fdDepW.Flush() } func depHeadParse(data []byte) (build uuid.UUID, tail []byte, err error) { if len(data) < len(BinMagic)+1+UUIDLen { err = errors.New("too short header") return } if !bytes.Equal(data[:len(BinMagic)], []byte(BinMagic)) { err = errors.New("bad magic") return } data = data[len(BinMagic):] switch data[0] { case BinVersionV1: default: err = errors.New("unknown version") return } build = uuid.Must(uuid.FromBytes(data[1 : 1+UUIDLen])) tail = data[1+UUIDLen:] return } func chunkRead(data []byte) (typ byte, chunk []byte, tail []byte, err error) { if len(data) < 2 { err = errors.New("no length") return } l := binary.BigEndian.Uint16(data[:2]) if l == 0 { err = errors.New("zero length chunk") return } if len(data) < int(l) { err = errors.New("not enough data") return } typ, chunk, tail = data[2], data[3:l], data[l:] return } func depBinIfchangeParse(tgt *Tgt, chunk []byte) (*Ifchange, string, error) { if len(chunk) < InodeLen+HashLen+1 { return nil, "", errors.New("too short \"ifchange\" format") } tgtH, _ := pathSplit(tgt.a) name := string(chunk[InodeLen+HashLen:]) ifchange := &Ifchange{ tgt: NewTgt(path.Join(tgtH, name)), meta: ([InodeLen + HashLen]byte)(chunk), } for _, cached := range IfchangeCache[ifchange.tgt.rel] { if ifchange.meta == cached.meta { return cached, name, nil } } if IfchangeCache != nil { IfchangeCache[ifchange.tgt.rel] = append(IfchangeCache[ifchange.tgt.rel], ifchange) } return ifchange, name, nil } func depParse(tgt *Tgt, data []byte) (*Dep, error) { build, data, err := depHeadParse(data) if err != nil { return nil, err } dep := Dep{build: build} var typ byte var chunk []byte for len(data) > 0 { typ, chunk, data, err = chunkRead(data) if err != nil { return nil, ErrLine(err) } switch typ { case DepTypeAlways: if len(chunk) != 0 { return nil, ErrLine(errors.New("bad \"always\" format")) } dep.always = true case DepTypeStamp: if len(chunk) != HashLen { return nil, ErrLine(errors.New("bad \"stamp\" format")) } dep.stamp = Hash(chunk) case DepTypeIfcreate: if len(chunk) < 1 { return nil, ErrLine(errors.New("too short \"ifcreate\" format")) } tgtH, _ := pathSplit(tgt.a) dep.ifcreates = append(dep.ifcreates, NewTgt(path.Join(tgtH, string(chunk)))) case DepTypeIfchange: ifchange, _, err := depBinIfchangeParse(tgt, chunk) if err != nil { return nil, ErrLine(err) } dep.ifchanges = append(dep.ifchanges, ifchange) case DepTypeIfchangeNonex: if len(chunk) < 1 { return nil, ErrLine(errors.New("too short \"ifchange\" format")) } dep.ifchanges = append(dep.ifchanges, &Ifchange{tgt: NewTgt(string(chunk))}) default: return nil, ErrLine(errors.New("unknown type")) } } return &dep, nil } func depRead(tgt *Tgt) (*Dep, error) { data, err := os.ReadFile(tgt.dep) if err != nil { return nil, err } return depParse(tgt, data) } func depReadOnlyIfchanges(pth string) (ifchanges []string, err error) { data, err := os.ReadFile(pth) if err != nil { return } _, data, err = depHeadParse(data) if err != nil { return nil, err } var typ byte var chunk []byte var tgt string tgtDummy := NewTgt("") for len(data) > 0 { typ, chunk, data, err = chunkRead(data) if err != nil { return nil, ErrLine(err) } switch typ { case DepTypeIfchange: _, tgt, err = depBinIfchangeParse(tgtDummy, chunk) if err != nil { return nil, ErrLine(err) } ifchanges = append(ifchanges, tgt) case DepTypeIfchangeNonex: ifchanges = append(ifchanges, string(chunk)) } } return } func depBuildRead(pth string) (uuid.UUID, error) { fd, err := os.Open(pth) if err != nil { return NullUUID, err } data := make([]byte, len(BinMagic)+1+UUIDLen) _, err = io.ReadFull(fd, data) fd.Close() if err != nil { return NullUUID, err } build, _, err := depHeadParse(data) return build, err } func depBuildWrite(w io.Writer, build uuid.UUID) (err error) { _, err = io.Copy(w, bytes.NewBuffer(bytes.Join([][]byte{ []byte(BinMagic), {BinVersionV1}, build[:], }, nil))) return }