X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=run.go;h=8ea91d1eb9ca10d5789cd5cd9159c6c9aeb715f4;hb=refs%2Fheads%2Fmaster;hp=a3e0fd65fb4ff8fc28d8805c1b91f073d49d2426;hpb=425dcfb13f3a9c9d4d189284e2991edf557f80db;p=goredo.git diff --git a/run.go b/run.go index a3e0fd6..8ea91d1 100644 --- a/run.go +++ b/run.go @@ -1,19 +1,17 @@ -/* -goredo -- djb's redo implementation on pure Go -Copyright (C) 2020-2023 Sergey Matveev - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ +// goredo -- djb's redo implementation on pure Go +// Copyright (C) 2020-2024 Sergey Matveev +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . // Targets runner @@ -21,7 +19,6 @@ package main import ( "bufio" - "bytes" "crypto/rand" "encoding/hex" "errors" @@ -58,7 +55,7 @@ const ( RedoDir = ".redo" LockSuffix = ".lock" - DepSuffix = ".rec" + DepSuffix = ".dep" TmpPrefix = ".redo." LogSuffix = ".log" LogRecSuffix = ".log-rec" @@ -102,7 +99,7 @@ func init() { } type RunError struct { - Tgt string + Tgt *Tgt DoFile string Started *time.Time Finished *time.Time @@ -112,7 +109,7 @@ type RunError struct { func (e *RunError) Name() string { var name string if e.DoFile == "" { - name = e.Tgt + name = e.Tgt.rel } else { name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile) } @@ -133,17 +130,17 @@ func mkdirs(pth string) error { return os.MkdirAll(pth, os.FileMode(0777)) } -func isModified(depInfo *DepInfo, cwd, tgt string) ( - modified bool, ourInode *Inode, hshPrev []byte, err error, +func isModified(dep *Dep, tgt *Tgt) ( + modified bool, ourInode *Inode, hshPrev Hash, err error, ) { - if depInfo == nil { + if dep == nil { return } - for _, dep := range depInfo.ifchanges { - if dep.tgt != tgt { + for _, ifchange := range dep.ifchanges { + if ifchange.tgt.a != tgt.a { continue } - ourInode, err = inodeFromFileByPath(path.Join(cwd, tgt)) + ourInode, err = inodeFromFileByPath(tgt.a) if err != nil { if os.IsNotExist(err) { err = nil @@ -152,8 +149,8 @@ func isModified(depInfo *DepInfo, cwd, tgt string) ( err = ErrLine(err) return } - hshPrev = dep.hash - modified = !ourInode.Equals(dep.inode) + hshPrev = ifchange.Hash() + modified = !ourInode.Equals(ifchange.Inode()) break } return @@ -169,14 +166,14 @@ func syncDir(dir string) error { return err } -func runScript(tgtOrig string, errs chan error, forced, traced bool) error { - cwd, tgt := cwdAndTgt(tgtOrig) - redoDir := path.Join(cwd, RedoDir) +func runScript(tgt *Tgt, errs chan error, forced, traced bool) error { + tgtH, tgtT := pathSplit(tgt.a) + redoDir := path.Join(tgtH, RedoDir) if err := mkdirs(redoDir); err != nil { - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } - shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgtOrig, cwd) + shCtx := fmt.Sprintf("sh: %s: cwd:%s", tgt, tgtH) jsToken := jsAcquire(shCtx) jsNeedsRelease := true defer func() { @@ -187,12 +184,12 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { // Acquire lock fdLock, err := os.OpenFile( - path.Join(redoDir, tgt+LockSuffix), + path.Join(redoDir, tgtT+LockSuffix), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.FileMode(0666), ) if err != nil { - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } flock := unix.Flock_t{ Type: unix.F_WRLCK, @@ -212,13 +209,13 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { if err = unix.FcntlFlock(fdLock.Fd(), unix.F_SETLK, &flock); err != nil { if uintptr(err.(syscall.Errno)) != uintptr(unix.EAGAIN) { fdLock.Close() - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } Jobs.Add(1) if err = unix.FcntlFlock(fdLock.Fd(), unix.F_GETLK, &flock); err != nil { log.Fatalln(err, fdLock.Name()) } - tracef(CDebug, "waiting: %s (pid=%d)", tgtOrig, flock.Pid) + tracef(CDebug, "waiting: %s (pid=%d)", tgt, flock.Pid) if FdStatus != nil { if _, err = FdStatus.Write([]byte{StatusWait}); err != nil { log.Fatal(err) @@ -236,123 +233,135 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { log.Fatalln(err, fdLock.Name()) } lockRelease() - tracef(CDebug, "waiting done: %s", tgtOrig) + tracef(CDebug, "waiting done: %s", tgt) if FdStatus != nil { if _, err = FdStatus.Write([]byte{StatusWaited}); err != nil { log.Fatal(err) } } - build, err := depReadBuild(path.Join(redoDir, tgt+DepSuffix)) + build, err := depBuildRead(tgt.dep) if err == nil { if build != BuildUUID { err = errors.New("was not built: build differs") } } else { if errors.Is(err, fs.ErrNotExist) { - err = errors.New("was not built: no .rec") + err = errors.New("was not built: no .dep") } } if err != nil { - err = TgtError{tgtOrig, err} + err = TgtError{tgt, err} } errs <- err }() return nil } - // It scans the whole .rec file while searching for the single target, + // It scans the whole .dep file while searching for the single target, // but that one is always located at the very end - depInfo, err := depRead(path.Join(redoDir, tgt+DepSuffix)) + dep, err := depRead(tgt) if err != nil { if errors.Is(err, fs.ErrNotExist) { err = nil } else { lockRelease() - return TgtError{tgtOrig, err} + return TgtError{tgt, ErrLine(err)} } } // Check if it was already built in parallel - if !forced && depInfo != nil && depInfo.build == BuildUUID { + if !forced && dep != nil && dep.build == BuildUUID { lockRelease() errs <- nil return nil } // Check if target is not modified externally - modified, inodePrev, hshPrev, err := isModified(depInfo, cwd, tgt) + modified, inodePrev, hshPrev, err := isModified(dep, tgt) if err != nil { lockRelease() - return TgtError{tgtOrig, err} + return TgtError{tgt, ErrLine(err)} } if modified { lockRelease() if StopIfMod { - return fmt.Errorf("%s externally modified", tgtOrig) + return fmt.Errorf("%s externally modified", tgt) } - tracef(CWarn, "%s externally modified: not redoing", tgtOrig) + Jobs.Add(1) + tracef(CWarn, "%s externally modified: not redoing", tgt) go func() { errs <- nil + Jobs.Done() }() return nil } - depInfo = nil + dep = nil - // Start preparing .rec - fdDep, err := tempfile(redoDir, tgt+DepSuffix) + // Start preparing .dep + fdDep, err := tempfile(redoDir, tgtT+DepSuffix) if err != nil { lockRelease() - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } fdDepOpened := true - fdDepPath := fdDep.Name() + fdDepExists := true + fdDepW := bufio.NewWriter(fdDep) cleanup := func() { lockRelease() if fdDepOpened { fdDep.Close() } - os.Remove(fdDep.Name()) - } - if _, err = recfile.NewWriter(fdDep).WriteFields( - recfile.Field{Name: "Build", Value: BuildUUID}, - ); err != nil { - cleanup() - return TgtError{tgtOrig, ErrLine(err)} - } - - // Find .do - doFile, upLevels, err := findDo(fdDep, cwd, tgt) - if err != nil { - cleanup() - return TgtError{tgtOrig, ErrLine(err)} + if fdDepExists { + os.Remove(fdDep.Name()) + } } - if doFile == "" { + if err = depBuildWrite(fdDepW, BuildUUID); err != nil { cleanup() - return TgtError{tgtOrig, errors.New("no .do found")} + return TgtError{tgt, ErrLine(err)} } + var cwd string + var dirPrefix string + var doFile *Tgt + var doFileT string + basename := tgtT + runErr := RunError{Tgt: tgt} // Determine basename and DIRPREFIX - doFileRelPath := doFile - ents := strings.Split(cwd, "/") - ents = ents[len(ents)-upLevels:] - dirPrefix := path.Join(ents...) - cwdOrig := cwd - for i := 0; i < upLevels; i++ { - cwd = path.Join(cwd, "..") - doFileRelPath = path.Join("..", doFileRelPath) - } - cwd = path.Clean(cwd) - doFilePath := path.Join(cwd, doFile) - basename := tgt - runErr := RunError{Tgt: tgtOrig} - if strings.HasPrefix(doFile, "default.") { - basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1] - runErr.DoFile = doFileRelPath + { + doFileRelPath, upLevels, err := findDo(fdDepW, fdDep.Name(), tgtH, tgtT) + if err != nil { + cleanup() + return TgtError{tgt, ErrLine(err)} + } + if doFileRelPath == "" { + cleanup() + return TgtError{tgt, errors.New("no .do found")} + } + ents := strings.Split(tgtH, "/") + ents = ents[len(ents)-upLevels:] + dirPrefix = path.Join(ents...) + ups := make([]string, 0, upLevels+2) + ups = append(ups, tgtH) + for i := 0; i < upLevels; i++ { + ups = append(ups, "..") + } + ups = append(ups, doFileRelPath) + cwd = path.Clean(path.Join(ups[:len(ups)-1]...)) + doFile = NewTgt(path.Join(ups...)) + doFileT = path.Base(doFile.a) + if strings.HasPrefix(doFileT, "default.") { + basename = basename[:len(basename)-(len(doFileT)-len("default.")-len(".do"))-1] + runErr.DoFile = doFile.rel + } } - if err = depWrite(fdDep, cwdOrig, doFileRelPath, nil); err != nil { + if err = depWrite(fdDepW, fdDep.Name(), tgtH, doFile, ""); err != nil { + cleanup() + return TgtError{tgt, ErrLine(err)} + } + if err = fdDepW.Flush(); err != nil { cleanup() - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } fdDep.Close() fdDepOpened = false @@ -361,8 +370,8 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { // Prepare command line var cmdName string var args []string - if err = unix.Access(doFilePath, unix.X_OK); err == nil { - cmdName = doFilePath + if err = unix.Access(doFile.rel, unix.X_OK); err == nil { + cmdName = doFile.a args = make([]string, 0, 3) } else { cmdName = "/bin/sh" @@ -371,28 +380,27 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { } else { args = append(args, "-e") } - args = append(args, doFile) + args = append(args, doFileT) } // Temporary file for stdout - fdStdout, err := tempfile(cwdOrig, tgt) + fdStdout, err := tempfile(tgtH, tgtT) if err != nil { cleanup() - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } - stdoutPath := fdStdout.Name() fdStdout.Close() - tmpPath := stdoutPath + ".3" // and for $3 + tmpPath := fdStdout.Name() + ".3" // and for $3 tmpPathRel := mustRel(cwd, tmpPath) args = append( args, - path.Join(dirPrefix, tgt), + path.Join(dirPrefix, tgtT), path.Join(dirPrefix, basename), tmpPathRel, ) shCtx = fmt.Sprintf( "sh: %s: %s %s cwd:%s dirprefix:%s", - tgtOrig, cmdName, args, cwd, dirPrefix, + tgt, cmdName, args, cwd, dirPrefix, ) cmd := exec.Command(cmdName, args...) @@ -431,13 +439,13 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { var fdStderr *os.File if StderrKeep { fdStderr, err = os.OpenFile( - path.Join(redoDir, tgt+LogSuffix), + path.Join(redoDir, tgtT+LogSuffix), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0666), ) if err != nil { cleanup() - return TgtError{tgtOrig, ErrLine(err)} + return TgtError{tgt, ErrLine(err)} } } tracef(CDebug, "%s", shCtx) @@ -472,7 +480,7 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { var exitErr *exec.ExitError started := time.Now() runErr.Started = &started - fdStdout, err = os.OpenFile(stdoutPath, os.O_RDWR, os.FileMode(0666)) + fdStdout, err = os.OpenFile(fdStdout.Name(), os.O_RDWR, os.FileMode(0666)) if err != nil { if fdStderr != nil { fdStderr.Close() @@ -482,7 +490,7 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { return } cmd.Stdout = fdStdout - fdDep, err = os.OpenFile(fdDepPath, os.O_WRONLY|os.O_APPEND, os.FileMode(0666)) + fdDep, err = os.OpenFile(fdDep.Name(), os.O_WRONLY|os.O_APPEND, os.FileMode(0666)) if err != nil { if fdStderr != nil { fdStderr.Close() @@ -493,6 +501,7 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { return } fdDepOpened = true + fdDepW.Reset(fdDep) cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum)) fdNum++ @@ -505,14 +514,14 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { fdStdout.Close() if fdStderr != nil { fdStderr.Close() - logRecPath := path.Join(redoDir, tgt+LogRecSuffix) + logRecPath := path.Join(redoDir, tgtT+LogRecSuffix) if fdStderr, err = os.OpenFile( logRecPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0666), ); err == nil { fields := []recfile.Field{ - {Name: "Build", Value: BuildUUID}, + {Name: "Build", Value: BuildUUID.String()}, {Name: "PPID", Value: strconv.Itoa(os.Getpid())}, {Name: "Cwd", Value: cwd}, } @@ -546,18 +555,21 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { Value: strconv.Itoa(exitErr.ProcessState.ExitCode()), }) } - w := bufio.NewWriter(fdStderr) - depInfo, err := depRead(fdDepPath) - if err != nil { - err = ErrLine(err) - goto Err - } - for _, dep := range depInfo.ifchanges { - fields = append(fields, recfile.Field{ - Name: "Ifchange", - Value: dep.tgt, - }) + w := bufio.NewWriter(fdStderr) + if !fdDepExists { + var ifchanges []string + ifchanges, err = depReadOnlyIfchanges(tgt.dep) + if err != nil { + err = ErrLine(err) + goto Err + } + for _, ifchange := range ifchanges { + fields = append(fields, recfile.Field{ + Name: "Ifchange", + Value: ifchange, + }) + } } _, err = recfile.NewWriter(w).WriteFields(fields...) if err != nil { @@ -576,7 +588,9 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { } } lockRelease() - os.Remove(fdDep.Name()) + if fdDepExists { + os.Remove(fdDep.Name()) + } os.Remove(fdStdout.Name()) os.Remove(tmpPath) if FdStatus != nil { @@ -603,7 +617,7 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { RunningProcs[cmd.Process.Pid] = cmd.Process RunningProcsM.Unlock() pid := fmt.Sprintf("[%d]", cmd.Process.Pid) - tracef(CDebug, "%s runs %s", tgtOrig, pid) + tracef(CDebug, "%s runs %s", tgt, pid) stderrTerm := make(chan struct{}) go func() { @@ -614,22 +628,27 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { line = scanner.Text() if strings.HasPrefix(line, childStderrPrefix) { line = line[len(childStderrPrefix):] - os.Stderr.WriteString(StderrPrefix + line + "\n") + if StderrPrefix == "" { + withPrependedTgt(line) + } else { + stderrWrite(StderrPrefix + line + "\n") + } continue } if fdStderr != nil { ts.FromTime(time.Now()) - LogMutex.Lock() fmt.Fprintln(fdStderr, tai64n.Encode(ts[:]), line) - LogMutex.Unlock() } if StderrSilent { continue } - if MyPid == 0 { - tracef(CNone, "%s", line) + if MyPID != 0 { + line = pid + " " + line + } + if StderrPrefix == "" { + withPrependedTgt("[" + tgt.rel + "]" + line) } else { - tracef(CNone, "%s %s", pid, line) + stderrWrite(StderrPrefix + "[" + tgt.rel + "]" + line + "\n") } } close(stderrTerm) @@ -651,17 +670,12 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { } // Was $1 touched? - if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil { + if inode, err := inodeFromFileByPath(tgt.a); err == nil { if inodePrev == nil { runErr.Err = Err1WasTouched errs <- runErr return } - if err != nil { - runErr.Err = err - errs <- runErr - return - } if !inode.Equals(inodePrev) { runErr.Err = Err1WasTouched errs <- runErr @@ -669,16 +683,6 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { } } - if inodePrev != nil { - if inode, err := inodeFromFileByPath(path.Join(cwdOrig, tgt)); err == nil { - if !inode.Equals(inodePrev) { - runErr.Err = Err1WasTouched - errs <- runErr - return - } - } - } - // Does it produce both stdout and tmp? fiStdout, err := os.Stat(fdStdout.Name()) if err != nil { @@ -703,6 +707,7 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { // Determine what file we must process at last var fd *os.File + var chmod fs.FileMode if tmpExists { fd, err = os.Open(tmpPath) if err != nil { @@ -710,21 +715,27 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { goto Finish } defer fd.Close() + if fi, rerr := fd.Stat(); rerr == nil { + chmod = fi.Mode() + } else { + err = ErrLine(rerr) + goto Finish + } } else if fiStdout.Size() > 0 { fd = fdStdout } // Do we need to ifcreate it, or ifchange with renaming? if fd == nil { - os.Remove(path.Join(cwdOrig, tgt)) - err = ifcreate(fdDep, tgt) + os.Remove(tgt.a) + err = ifcreate(fdDepW, fdDep.Name(), tgtT) if err != nil { err = ErrLine(err) goto Finish } } else { - var hsh []byte - if hshPrev != nil { + var hsh Hash + if hshPrev != "" { _, err = fd.Seek(0, io.SeekStart) if err != nil { err = ErrLine(err) @@ -735,23 +746,29 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { err = ErrLine(err) goto Finish } - if bytes.Equal(hsh, hshPrev) { - tracef(CDebug, "%s has same hash, not renaming", tgtOrig) + if hsh == hshPrev { + tracef(CDebug, "%s has same hash, not renaming", tgt) err = ErrLine(os.Remove(fd.Name())) if err != nil { goto Finish } - err = ErrLine(os.Chtimes(path.Join(cwdOrig, tgt), finished, finished)) + if chmod != 0 { + err = ErrLine(os.Chmod(tgt.a, chmod)) + if err != nil { + goto Finish + } + } + err = ErrLine(os.Chtimes(tgt.a, finished, finished)) if err != nil { goto Finish } if !NoSync { - err = ErrLine(syncDir(cwdOrig)) + err = ErrLine(syncDir(tgtH)) if err != nil { goto Finish } } - err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh)) + err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtH, tgt, hsh)) if err != nil { goto Finish } @@ -764,35 +781,39 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { goto Finish } } - err = ErrLine(os.Rename(fd.Name(), path.Join(cwdOrig, tgt))) + err = ErrLine(os.Rename(fd.Name(), tgt.a)) if err != nil { goto Finish } if !NoSync { - err = ErrLine(syncDir(cwdOrig)) + err = ErrLine(syncDir(tgtH)) if err != nil { goto Finish } } - err = ErrLine(depWrite(fdDep, cwdOrig, tgt, hsh)) + err = ErrLine(depWrite(fdDepW, fdDep.Name(), tgtH, tgt, hsh)) if err != nil { goto Finish } } RecCommit: - // Commit .rec + // Commit .dep + err = ErrLine(fdDepW.Flush()) + if err != nil { + goto Finish + } if !NoSync { err = ErrLine(fdDep.Sync()) if err != nil { goto Finish } } - fdDepPath = path.Join(redoDir, tgt+DepSuffix) - err = ErrLine(os.Rename(fdDep.Name(), fdDepPath)) + err = ErrLine(os.Rename(fdDep.Name(), tgt.dep)) if err != nil { goto Finish } + fdDepExists = false if !NoSync { err = ErrLine(syncDir(redoDir)) if err != nil { @@ -802,14 +823,15 @@ func runScript(tgtOrig string, errs chan error, forced, traced bool) error { fdDep.Close() fdDepOpened = false - // Post-commit .rec sanitizing - if depInfo, err := depRead(fdDepPath); err == nil { - ifchangeSeen := make(map[string]struct{}, len(depInfo.ifchanges)) - for _, dep := range depInfo.ifchanges { - ifchangeSeen[dep.tgt] = struct{}{} + // Post-commit .dep sanitizing + dep, err = depRead(tgt) + if err == nil { + ifchangeSeen := make(map[string]struct{}, len(dep.ifchanges)) + for _, ifchange := range dep.ifchanges { + ifchangeSeen[ifchange.tgt.a] = struct{}{} } - for _, dep := range depInfo.ifcreates { - if _, exists := ifchangeSeen[dep]; exists { + for _, ifcreate := range dep.ifcreates { + if _, exists := ifchangeSeen[ifcreate.a]; exists { tracef(CWarn, "simultaneous ifcreate and ifchange records: %s", tgt) } }