X-Git-Url: http://www.git.cypherpunks.ru/?p=goredo.git;a=blobdiff_plain;f=run.go;h=af71b3b8b3b4c4b08d39082e295d2fa48316ed62;hp=58b9f6daff2fa933a17c6ff906766676169c86dd;hb=ce96a1c785b32af13264225c0bf7ae8370a5af21;hpb=bc7701e7a4f95cee680e0736ec3e68a8b0b5c09f diff --git a/run.go b/run.go index 58b9f6d..af71b3b 100644 --- a/run.go +++ b/run.go @@ -1,6 +1,6 @@ /* -goredo -- redo implementation on pure Go -Copyright (C) 2020 Sergey Matveev +goredo -- djb's redo implementation on pure Go +Copyright (C) 2020-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,64 +21,104 @@ package main import ( "bufio" - "encoding/hex" "errors" + "flag" "fmt" "io" "os" "os/exec" "path" - "strconv" + "path/filepath" "strings" + "sync" "syscall" "time" "go.cypherpunks.ru/recfile" + "go.cypherpunks.ru/tai64n" "golang.org/x/sys/unix" ) const ( + EnvDepFd = "REDO_DEP_FD" + EnvDirPrefix = "REDO_DIRPREFIX" + EnvBuildUUID = "REDO_BUILD_UUID" + EnvStderrPrefix = "REDO_STDERR_PREFIX" + EnvTrace = "REDO_TRACE" + EnvStderrKeep = "REDO_LOGS" + EnvStderrSilent = "REDO_SILENT" + EnvNoSync = "REDO_NO_SYNC" + RedoDir = ".redo" LockSuffix = ".lock" - DepSuffix = ".dep" + DepSuffix = ".rec" TmpPrefix = ".redo." LogSuffix = ".log" ) -func mkdirs(pth string) error { - if _, err := os.Stat(pth); err == nil { - return nil +var ( + NoSync bool = false + StderrKeep bool = false + StderrSilent bool = false + StderrPrefix string + Jobs sync.WaitGroup + + flagTrace = flag.Bool("x", false, "trace (sh -x) current targets") + flagTraceAll = flag.Bool("xx", false, fmt.Sprintf("trace (sh -x) all targets (%s=1)", EnvTrace)) + flagStderrKeep = flag.Bool("logs", false, fmt.Sprintf("keep job's stderr (%s=1)", EnvStderrKeep)) + flagStderrSilent = flag.Bool("silent", false, fmt.Sprintf("do not print job's stderr (%s=1)", EnvStderrSilent)) +) + +type RunErr struct { + Tgt string + DoFile string + Started *time.Time + Finished *time.Time + Err error +} + +func (e *RunErr) Name() string { + var name string + if e.DoFile == "" { + name = e.Tgt + } else { + name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile) } - return os.MkdirAll(pth, os.FileMode(0777)) + if e.Finished == nil { + return name + } + return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds()) } -func tempsuffix() string { - return strconv.FormatInt((time.Now().UnixNano()+int64(os.Getpid()))&0xFFFFFFFF, 16) +func (e RunErr) Error() string { + return fmt.Sprintf("%s: %s", e.Name(), e.Err) } -func tempfile(dir, prefix string) (*os.File, error) { - // It respects umask, unlike ioutil.TempFile - name := path.Join(dir, TmpPrefix+prefix+"."+tempsuffix()) - return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666)) +func mkdirs(pth string) error { + if _, err := os.Stat(pth); err == nil { + return nil + } + return os.MkdirAll(pth, os.FileMode(0777)) } -func isModified(cwd, redoDir, tgt string) (bool, error) { +func isModified(cwd, redoDir, tgt string) (bool, *Inode, error) { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - return false, nil + return false, nil, nil } - return false, err + return false, nil, err } defer fdDep.Close() r := recfile.NewReader(fdDep) + var ourInode *Inode for { m, err := r.NextMap() if err != nil { if err == io.EOF { break } - return false, err + return false, nil, err } if m["Target"] != tgt { continue @@ -86,48 +126,25 @@ func isModified(cwd, redoDir, tgt string) (bool, error) { fd, err := os.Open(path.Join(cwd, tgt)) if err != nil { if os.IsNotExist(err) { - return false, nil + return false, nil, nil } - return false, err + return false, nil, err + } + ourInode, err = inodeFromFile(fd) + fd.Close() + if err != nil { + return false, nil, err } - defer fd.Close() - ourTs, err := fileCtime(fd) + theirInode, err := inodeFromRec(m) if err != nil { - return false, err + return false, nil, err } - if ourTs != m["Ctime"] { - return true, nil + if !ourInode.Equals(theirInode) { + return true, ourInode, nil } break } - return false, nil -} - -type RunErr struct { - Tgt string - DoFile string - Started *time.Time - Finished *time.Time - Err error -} - -func (e RunErr) Unwrap() error { return e.Err } - -func (e *RunErr) Name() string { - var name string - if e.DoFile == "" { - name = e.Tgt - } else { - name = fmt.Sprintf("%s (%s)", e.Tgt, e.DoFile) - } - if e.Finished == nil { - return name - } - return fmt.Sprintf("%s (%fsec)", name, e.Finished.Sub(*e.Started).Seconds()) -} - -func (e RunErr) Error() string { - return fmt.Sprintf("%s: %s", e.Name(), e.Err) + return false, ourInode, nil } func syncDir(dir string) error { @@ -140,9 +157,8 @@ func syncDir(dir string) error { return err } -func runScript(tgt string, errs chan error) error { - tgtOrig := tgt - cwd, tgt := cwdAndTgt(tgt) +func runScript(tgtOrig string, errs chan error, traced bool) error { + cwd, tgt := cwdAndTgt(tgtOrig) redoDir := path.Join(cwd, RedoDir) if err := mkdirs(redoDir); err != nil { return TgtErr{tgtOrig, err} @@ -170,28 +186,35 @@ func runScript(tgt string, errs chan error) error { fdLock.Close() return TgtErr{tgtOrig, err} } - trace(CDebug, "waiting: %s", tgtOrig) Jobs.Add(1) + trace(CDebug, "waiting: %s", tgtOrig) + if FdStatus != nil { + FdStatus.Write([]byte{StatusWait}) + } go func() { defer Jobs.Done() trace(CLock, "LOCK_EX: %s", fdLock.Name()) unix.Flock(int(fdLock.Fd()), unix.LOCK_EX) lockRelease() trace(CDebug, "waiting done: %s", tgtOrig) - var builtNow bool + if FdStatus != nil { + FdStatus.Write([]byte{StatusWaited}) + } + var depInfo *DepInfo fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - err = errors.New("was not built") + err = errors.New("was not built: no .rec") } goto Finish } - builtNow, _, err = isBuiltNow(fdDep) + defer fdDep.Close() + depInfo, err = depRead(fdDep) if err != nil { goto Finish } - if !builtNow { - err = errors.New("was not built") + if depInfo.build != BuildUUID { + err = errors.New("was not built: build differs") } Finish: if err != nil { @@ -203,7 +226,7 @@ func runScript(tgt string, errs chan error) error { } // Check if target is not modified externally - modified, err := isModified(cwd, redoDir, tgt) + modified, inodePrev, err := isModified(cwd, redoDir, tgt) if err != nil { lockRelease() return TgtErr{tgtOrig, err} @@ -211,16 +234,19 @@ func runScript(tgt string, errs chan error) error { if modified { trace(CWarn, "%s externally modified: not redoing", tgtOrig) lockRelease() - errs <- nil - return TgtErr{tgtOrig, err} + go func() { + errs <- nil + }() + return nil } - // Start preparing .dep + // Start preparing .rec fdDep, err := tempfile(redoDir, tgt+DepSuffix) if err != nil { lockRelease() return TgtErr{tgtOrig, err} } + fdDepPath := fdDep.Name() cleanup := func() { lockRelease() fdDep.Close() @@ -243,112 +269,110 @@ func runScript(tgt string, errs chan error) error { cleanup() return TgtErr{tgtOrig, errors.New("no .do found")} } - if err = writeDep(fdDep, cwd, doFile); err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } // Determine basename and DIRPREFIX + doFileRelPath := doFile ents := strings.Split(cwd, "/") ents = ents[len(ents)-upLevels:] dirPrefix := path.Join(ents...) cwdOrig := cwd for i := 0; i < upLevels; i++ { cwd = path.Join(cwd, "..") + doFileRelPath = path.Join("..", doFileRelPath) } cwd = path.Clean(cwd) + doFilePath := path.Join(cwd, doFile) basename := tgt runErr := RunErr{Tgt: tgtOrig} if strings.HasPrefix(doFile, "default.") { basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1] - runErr.DoFile = doFile + runErr.DoFile = doFileRelPath } + + if err = writeDep(fdDep, cwdOrig, doFileRelPath); err != nil { + cleanup() + return TgtErr{tgtOrig, err} + } + fdDep.Close() trace(CWait, "%s", runErr.Name()) - doFile = path.Base(doFile) // Prepare command line var cmdName string var args []string - if err = unix.Access(path.Join(cwd, doFile), unix.X_OK); err == nil { - // Ordinary executable file - cmdName = doFile + if err = unix.Access(doFilePath, unix.X_OK); err == nil { + cmdName = doFilePath args = make([]string, 0, 3) } else { - fd, err := os.Open(path.Join(cwd, doFile)) - if err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } - buf := make([]byte, 512) - n, err := fd.Read(buf) - if err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } - if n > 3 && string(buf[:3]) == "#!/" { - // Shebanged - t := string(buf[2:n]) - nlIdx := strings.Index(t, "\n") - if nlIdx == -1 { - cleanup() - return TgtErr{tgtOrig, errors.New("not fully read shebang")} - } - args = strings.Split(t[:nlIdx], " ") - cmdName, args = args[0], args[1:] + cmdName = "/bin/sh" + if traced { + args = append(args, "-ex") } else { - // Shell - cmdName = "/bin/sh" - if Trace { - args = append(args, "-ex") - } else { - args = append(args, "-e") - } + args = append(args, "-e") } args = append(args, doFile) } // Temporary file for stdout - fdStdout, err := tempfile(cwd, tgt) + fdStdout, err := tempfile(cwdOrig, tgt) if err != nil { cleanup() return TgtErr{tgtOrig, err} } - tmpPath := fdStdout.Name() + ".3" // and for $3 - args = append(args, tgt, basename, path.Base(tmpPath)) + stdoutPath := fdStdout.Name() + fdStdout.Close() + tmpPath := stdoutPath + ".3" // and for $3 + tmpPathRel, err := filepath.Rel(cwd, tmpPath) + if err != nil { + panic(err) + } + args = append( + args, + path.Join(dirPrefix, tgt), + path.Join(dirPrefix, basename), + tmpPathRel, + ) cmd := exec.Command(cmdName, args...) cmd.Dir = cwd - cmd.Stdout = fdStdout - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", RedoLevelEnv, Level+1)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoDirPrefixEnv, dirPrefix)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoBuildUUIDEnv, BuildUUID)) + // cmd.Stdin reads from /dev/null by default + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID)) + childStderrPrefix := tempsuffix() cmd.Env = append(cmd.Env, fmt.Sprintf( - "%s=%s", RedoStderrPrefixEnv, childStderrPrefix, + "%s=%s", EnvStderrPrefix, childStderrPrefix, )) - cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) fdNum := 0 - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoDepFdEnv, 3+fdNum)) + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum)) + fdNum++ + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum)) fdNum++ + + if FdStatus == nil { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvStatusFd)) + } else { + cmd.ExtraFiles = append(cmd.ExtraFiles, FdStatus) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvStatusFd, 3+fdNum)) + fdNum++ + } + if JSR == nil { // infinite jobs - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSRFdEnv)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSWFdEnv)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", EnvJSFd)) } else { cmd.ExtraFiles = append(cmd.ExtraFiles, JSR) cmd.ExtraFiles = append(cmd.ExtraFiles, JSW) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSRFdEnv, 3+fdNum)) - fdNum++ - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSWFdEnv, 3+fdNum)) - fdNum++ + cmd.Env = append(cmd.Env, fmt.Sprintf( + "%s=%d,%d", EnvJSFd, 3+fdNum+0, 3+fdNum+1, + )) + fdNum += 2 } // Preparing stderr - stderr, err := cmd.StderrPipe() - if err != nil { - panic(err) - } var fdStderr *os.File if StderrKeep { fdStderr, err = os.OpenFile( @@ -362,12 +386,38 @@ func runScript(tgt string, errs chan error) error { } fdStderr.Truncate(0) } - shCtx := fmt.Sprintf("sh: %s: %s %s [%s]", tgtOrig, cmdName, args, cwd) + shCtx := fmt.Sprintf( + "sh: %s: %s %s cwd:%s dirprefix:%s", + tgtOrig, cmdName, args, cwd, dirPrefix, + ) trace(CDebug, "%s", shCtx) Jobs.Add(1) go func() { jsAcquire(shCtx) + if FdStatus != nil { + FdStatus.Write([]byte{StatusRun}) + } + + started := time.Now() + runErr.Started = &started + fdStdout, err = os.OpenFile(stdoutPath, os.O_RDWR, os.FileMode(0666)) + if err != nil { + runErr.Err = err + errs <- runErr + return + } + cmd.Stdout = fdStdout + fdDep, err = os.OpenFile(fdDepPath, os.O_WRONLY|os.O_APPEND, os.FileMode(0666)) + if err != nil { + runErr.Err = err + errs <- runErr + return + } + cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum)) + fdNum++ + defer func() { jsRelease(shCtx) lockRelease() @@ -380,11 +430,19 @@ func runScript(tgt string, errs chan error) error { os.Remove(fdStdout.Name()) os.Remove(tmpPath) os.Remove(fdLock.Name()) + if FdStatus != nil { + FdStatus.Write([]byte{StatusDone}) + } Jobs.Done() }() - started := time.Now() - runErr.Started = &started - err := cmd.Start() + stderr, err := cmd.StderrPipe() + if err != nil { + runErr.Err = err + errs <- runErr + return + } + started = time.Now() + err = cmd.Start() if err != nil { runErr.Err = err errs <- runErr @@ -397,7 +455,7 @@ func runScript(tgt string, errs chan error) error { go func() { scanner := bufio.NewScanner(stderr) var line string - ts := new(TAI64N) + ts := new(tai64n.TAI64N) for scanner.Scan() { line = scanner.Text() if strings.HasPrefix(line, childStderrPrefix) { @@ -406,8 +464,10 @@ func runScript(tgt string, errs chan error) error { continue } if fdStderr != nil { - tai64nNow(ts) - fmt.Fprintf(fdStderr, "@%s %s\n", hex.EncodeToString(ts[:]), line) + tai64n.FromTime(time.Now(), ts) + LogMutex.Lock() + fmt.Fprintf(fdStderr, "%s %s\n", ts.Encode(), line) + LogMutex.Unlock() } if StderrSilent { continue @@ -432,6 +492,19 @@ func runScript(tgt string, errs chan error) error { return } + // Was $1 touched? + if inodePrev != nil { + if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { + inode, err := inodeFromFile(fd) + fd.Close() + if err == nil && !inode.Equals(inodePrev) { + runErr.Err = errors.New("$1 was explicitly touched") + errs <- runErr + return + } + } + } + // Does it produce both stdout and tmp? fiStdout, err := os.Stat(fdStdout.Name()) if err != nil { @@ -468,6 +541,7 @@ func runScript(tgt string, errs chan error) error { // Do we need to ifcreate it, of ifchange with renaming? if fd == nil { + os.Remove(path.Join(cwdOrig, tgt)) err = ifcreate(fdDep, tgt) if err != nil { goto Finish @@ -495,7 +569,7 @@ func runScript(tgt string, errs chan error) error { } } - // Commit .dep + // Commit .rec if !NoSync { err = fdDep.Sync() if err != nil { @@ -518,3 +592,15 @@ func runScript(tgt string, errs chan error) error { }() return nil } + +func isOkRun(err error) bool { + if err == nil { + return true + } + if err, ok := err.(RunErr); ok && err.Err == nil { + trace(CRedo, "%s", err.Name()) + return true + } + trace(CErr, "%s", err) + return false +}