/*
goredo -- djb's redo implementation on pure Go
-Copyright (C) 2020-2022 Sergey Matveev <stargrave@stargrave.org>
+Copyright (C) 2020-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
import (
"bufio"
+ "bytes"
"crypto/rand"
"encoding/hex"
"errors"
"flag"
"fmt"
"io"
+ "io/fs"
"log"
"os"
"os/exec"
"path"
- "path/filepath"
"strconv"
"strings"
"sync"
return os.MkdirAll(pth, os.FileMode(0777))
}
-func isModified(cwd, redoDir, tgt string) (bool, *Inode, string, error) {
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
- if err != nil {
- if os.IsNotExist(err) {
- return false, nil, "", nil
- }
- return false, nil, "", err
+func isModified(depInfo *DepInfo, cwd, tgt string) (
+ modified bool, ourInode *Inode, hshPrev []byte, err error,
+) {
+ if depInfo == nil {
+ return
}
- defer fdDep.Close()
- r := recfile.NewReader(fdDep)
- var modified bool
- var ourInode *Inode
- var hshPrev string
- for {
- m, err := r.NextMap()
- if err != nil {
- if errors.Is(err, io.EOF) {
- break
- }
- return false, nil, "", err
- }
- if m["Type"] != DepTypeIfchange || m["Target"] != tgt {
+ for _, dep := range depInfo.ifchanges {
+ if dep.tgt != tgt {
continue
}
ourInode, err = inodeFromFileByPath(path.Join(cwd, tgt))
if err != nil {
if os.IsNotExist(err) {
- return false, nil, "", nil
+ err = nil
+ return
}
- return false, nil, "", err
- }
- theirInode, err := inodeFromRec(m)
- if err != nil {
- return false, nil, "", err
+ err = ErrLine(err)
+ return
}
- hshPrev = m["Hash"]
- modified = !ourInode.Equals(theirInode)
+ hshPrev = dep.hash
+ modified = !ourInode.Equals(dep.inode)
break
}
- return modified, ourInode, hshPrev, nil
+ return
}
func syncDir(dir string) error {
return err
}
-func runScript(tgtOrig string, errs chan error, traced bool) error {
+func runScript(tgtOrig string, errs chan error, forced, traced bool) error {
cwd, tgt := cwdAndTgt(tgtOrig)
redoDir := path.Join(cwd, RedoDir)
if err := mkdirs(redoDir); err != nil {
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgtOrig, cwd)
os.FileMode(0666),
)
if err != nil {
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
flock := unix.Flock_t{
Type: unix.F_WRLCK,
tracef(CLock, "LOCK_UN: %s", fdLock.Name())
flock.Type = unix.F_UNLCK
if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
- log.Fatalln(err)
+ log.Fatalln(err, fdLock.Name())
}
fdLock.Close()
}
if err = unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil {
if uintptr(err.(syscall.Errno)) != uintptr(unix.EAGAIN) {
fdLock.Close()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
Jobs.Add(1)
if err = unix.FcntlFlock(fdLock.Fd(), unix.F_GETLK, &flock); err != nil {
- log.Fatalln(err)
+ log.Fatalln(err, fdLock.Name())
}
tracef(CDebug, "waiting: %s (pid=%d)", tgtOrig, flock.Pid)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWait}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
go func() {
defer Jobs.Done()
tracef(CLock, "LOCK_EX: %s", fdLock.Name())
+ LockAgain:
if err := unix.FcntlFlock(fdLock.Fd(), unix.F_SETLKW, &flock); err != nil {
- log.Fatalln(err)
+ if errors.Is(err, unix.EDEADLK) {
+ time.Sleep(10 * time.Millisecond)
+ goto LockAgain
+ }
+ log.Fatalln(err, fdLock.Name())
}
lockRelease()
tracef(CDebug, "waiting done: %s", tgtOrig)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusWaited}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
- var depInfo *DepInfo
- fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
+ build, err := depReadBuild(path.Join(redoDir, tgt+DepSuffix))
if err != nil {
- if os.IsNotExist(err) {
+ if errors.Is(err, fs.ErrNotExist) {
err = errors.New("was not built: no .rec")
}
goto Finish
}
- defer fdDep.Close()
- depInfo, err = depRead(fdDep)
- if err != nil {
- goto Finish
- }
- if depInfo.build != BuildUUID {
+ if build != BuildUUID {
err = errors.New("was not built: build differs")
}
Finish:
return nil
}
+ depInfo, err := depRead(path.Join(redoDir, tgt+DepSuffix))
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ lockRelease()
+ return TgtError{tgtOrig, err}
+ }
+ }
+
+ // Check if it was already built in parallel
+ if !forced && depInfo != nil && depInfo.build == BuildUUID {
+ lockRelease()
+ errs <- nil
+ return nil
+ }
+
// Check if target is not modified externally
- modified, inodePrev, hshPrev, err := isModified(cwd, redoDir, tgt)
+ modified, inodePrev, hshPrev, err := isModified(depInfo, cwd, tgt)
if err != nil {
lockRelease()
return TgtError{tgtOrig, err}
}()
return nil
}
+ depInfo = nil
// Start preparing .rec
fdDep, err := tempfile(redoDir, tgt+DepSuffix)
if err != nil {
lockRelease()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
fdDepPath := fdDep.Name()
cleanup := func() {
recfile.Field{Name: "Build", Value: BuildUUID},
); err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
// Find .do
doFile, upLevels, err := findDo(fdDep, cwd, tgt)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
if doFile == "" {
cleanup()
runErr.DoFile = doFileRelPath
}
- if err = depWrite(fdDep, cwdOrig, doFileRelPath, ""); err != nil {
+ if err = depWrite(fdDep, cwdOrig, doFileRelPath, nil); err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
fdDep.Close()
tracef(CWait, "%s", runErr.Name())
fdStdout, err := tempfile(cwdOrig, tgt)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
stdoutPath := fdStdout.Name()
fdStdout.Close()
tmpPath := stdoutPath + ".3" // and for $3
- tmpPathRel, err := filepath.Rel(cwd, tmpPath)
- if err != nil {
- panic(err)
- }
+ tmpPathRel := mustRel(cwd, tmpPath)
args = append(
args,
path.Join(dirPrefix, tgt),
)
if err != nil {
cleanup()
- return TgtError{tgtOrig, err}
+ return TgtError{tgtOrig, ErrLine(err)}
}
}
tracef(CDebug, "%s", shCtx)
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusRun}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
}
w := bufio.NewWriter(fdStderr)
- var depInfo *DepInfo
- fdDep, err := os.Open(fdDepPath)
- if err != nil {
- goto Err
- }
- depInfo, err = depRead(fdDep)
- fdDep.Close()
+ depInfo, err := depRead(fdDepPath)
if err != nil {
+ err = ErrLine(err)
goto Err
}
for _, dep := range depInfo.ifchanges {
fields = append(fields, recfile.Field{
Name: "Ifchange",
- Value: dep["Target"],
+ Value: dep.tgt,
})
}
_, err = recfile.NewWriter(w).WriteFields(fields...)
if err != nil {
+ err = ErrLine(err)
goto Err
}
- err = w.Flush()
+ err = ErrLine(w.Flush())
Err:
if err != nil {
- log.Println(err)
+ log.Print(err)
os.Remove(logRecPath)
}
fdStderr.Close()
os.Remove(fdDep.Name())
os.Remove(fdStdout.Name())
os.Remove(tmpPath)
- os.Remove(fdLock.Name())
if FdStatus != nil {
if _, err = FdStatus.Write([]byte{StatusDone}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
Jobs.Done()
// Does it produce both stdout and tmp?
fiStdout, err := os.Stat(fdStdout.Name())
if err != nil {
- runErr.Err = err
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
return
}
tmpExists = true
- } else if !os.IsNotExist(err) {
- runErr.Err = err
+ } else if !errors.Is(err, fs.ErrNotExist) {
+ runErr.Err = ErrLine(err)
errs <- runErr
return
}
if tmpExists {
fd, err = os.Open(tmpPath)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
defer fd.Close()
os.Remove(path.Join(cwdOrig, tgt))
err = ifcreate(fdDep, tgt)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
} else {
- var hsh string
- if hshPrev != "" {
+ var hsh []byte
+ if hshPrev != nil {
_, err = fd.Seek(0, io.SeekStart)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
hsh, err = fileHash(fd)
if err != nil {
+ err = ErrLine(err)
goto Finish
}
- if hsh == hshPrev {
+ if bytes.Equal(hsh, hshPrev) {
tracef(CDebug, "%s has same hash, not renaming", tgtOrig)
- err = os.Remove(fd.Name())
+ err = ErrLine(os.Remove(fd.Name()))
if err != nil {
goto Finish
}
- err = os.Chtimes(path.Join(cwdOrig, tgt), finished, finished)
+ err = ErrLine(os.Chtimes(path.Join(cwdOrig, tgt), finished, finished))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(cwdOrig)
+ err = ErrLine(syncDir(cwdOrig))
if err != nil {
goto Finish
}
}
- err = depWrite(fdDep, cwdOrig, tgt, hshPrev)
+ err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh))
if err != nil {
goto Finish
}
}
}
if !NoSync {
- err = fd.Sync()
+ err = ErrLine(fd.Sync())
if err != nil {
goto Finish
}
}
- err = os.Rename(fd.Name(), path.Join(cwdOrig, tgt))
+ err = ErrLine(os.Rename(fd.Name(), path.Join(cwdOrig, tgt)))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(cwdOrig)
+ err = ErrLine(syncDir(cwdOrig))
if err != nil {
goto Finish
}
}
- err = depWrite(fdDep, cwdOrig, tgt, hsh)
+ err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh))
if err != nil {
goto Finish
}
RecCommit:
// Commit .rec
if !NoSync {
- err = fdDep.Sync()
+ err = ErrLine(fdDep.Sync())
if err != nil {
goto Finish
}
}
fdDepPath = path.Join(redoDir, tgt+DepSuffix)
- err = os.Rename(fdDep.Name(), fdDepPath)
+ err = ErrLine(os.Rename(fdDep.Name(), fdDepPath))
if err != nil {
goto Finish
}
if !NoSync {
- err = syncDir(redoDir)
+ err = ErrLine(syncDir(redoDir))
if err != nil {
goto Finish
}
}
+ fdDep.Close()
// Post-commit .rec sanitizing
- fdDep.Close()
- if fdDepR, err := os.Open(fdDepPath); err == nil {
- depInfo, err := depRead(fdDepR)
- fdDepR.Close()
- if err != nil {
- goto Finish
- }
+ if depInfo, err := depRead(fdDepPath); err == nil {
ifchangeSeen := make(map[string]struct{}, len(depInfo.ifchanges))
for _, dep := range depInfo.ifchanges {
- ifchangeSeen[dep["Target"]] = struct{}{}
+ ifchangeSeen[dep.tgt] = struct{}{}
}
for _, dep := range depInfo.ifcreates {
if _, exists := ifchangeSeen[dep]; exists {
tracef(CWarn, "simultaneous ifcreate and ifchange records: %s", tgt)
}
}
+ } else if errors.Is(err, fs.ErrNotExist) {
+ err = nil
+ } else {
+ err = ErrLine(err)
+ goto Finish
}
Finish: