X-Git-Url: http://www.git.cypherpunks.ru/?p=goredo.git;a=blobdiff_plain;f=run.go;h=af71b3b8b3b4c4b08d39082e295d2fa48316ed62;hp=93c837d9fb458dca6c4bffbe62c120da60aa387f;hb=ce96a1c785b32af13264225c0bf7ae8370a5af21;hpb=66de71f560cd772e7ef9e9cf1182ce67e495b1ec diff --git a/run.go b/run.go index 93c837d..af71b3b 100644 --- a/run.go +++ b/run.go @@ -1,6 +1,6 @@ /* -goredo -- redo implementation on pure Go -Copyright (C) 2020 Sergey Matveev +goredo -- djb's redo implementation on pure Go +Copyright (C) 2020-2021 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,7 +21,6 @@ package main import ( "bufio" - "encoding/hex" "errors" "flag" "fmt" @@ -29,13 +28,14 @@ import ( "os" "os/exec" "path" - "strconv" + "path/filepath" "strings" "sync" "syscall" "time" "go.cypherpunks.ru/recfile" + "go.cypherpunks.ru/tai64n" "golang.org/x/sys/unix" ) @@ -51,7 +51,7 @@ const ( RedoDir = ".redo" LockSuffix = ".lock" - DepSuffix = ".dep" + DepSuffix = ".rec" TmpPrefix = ".redo." LogSuffix = ".log" ) @@ -63,7 +63,8 @@ var ( StderrPrefix string Jobs sync.WaitGroup - flagTrace = flag.Bool("x", false, fmt.Sprintf("trace current target (sh -x) (set %s=1 for others too)", EnvTrace)) + flagTrace = flag.Bool("x", false, "trace (sh -x) current targets") + flagTraceAll = flag.Bool("xx", false, fmt.Sprintf("trace (sh -x) all targets (%s=1)", EnvTrace)) flagStderrKeep = flag.Bool("logs", false, fmt.Sprintf("keep job's stderr (%s=1)", EnvStderrKeep)) flagStderrSilent = flag.Bool("silent", false, fmt.Sprintf("do not print job's stderr (%s=1)", EnvStderrSilent)) ) @@ -76,8 +77,6 @@ type RunErr struct { Err error } -func (e RunErr) Unwrap() error { return e.Err } - func (e *RunErr) Name() string { var name string if e.DoFile == "" { @@ -102,34 +101,24 @@ func mkdirs(pth string) error { return os.MkdirAll(pth, os.FileMode(0777)) } -func tempsuffix() string { - return strconv.FormatInt((time.Now().UnixNano()+int64(os.Getpid()))&0xFFFFFFFF, 16) -} - -func tempfile(dir, prefix string) (*os.File, error) { - // It respects umask, unlike ioutil.TempFile - name := path.Join(dir, TmpPrefix+prefix+"."+tempsuffix()) - return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666)) -} - -func isModified(cwd, redoDir, tgt string) (bool, string, error) { +func isModified(cwd, redoDir, tgt string) (bool, *Inode, error) { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err } defer fdDep.Close() r := recfile.NewReader(fdDep) - var ourTs string + var ourInode *Inode for { m, err := r.NextMap() if err != nil { if err == io.EOF { break } - return false, "", err + return false, nil, err } if m["Target"] != tgt { continue @@ -137,21 +126,25 @@ func isModified(cwd, redoDir, tgt string) (bool, string, error) { fd, err := os.Open(path.Join(cwd, tgt)) if err != nil { if os.IsNotExist(err) { - return false, "", nil + return false, nil, nil } - return false, "", err + return false, nil, err + } + ourInode, err = inodeFromFile(fd) + fd.Close() + if err != nil { + return false, nil, err } - defer fd.Close() - ourTs, err = fileCtime(fd) + theirInode, err := inodeFromRec(m) if err != nil { - return false, "", err + return false, nil, err } - if ourTs != m["Ctime"] { - return true, ourTs, nil + if !ourInode.Equals(theirInode) { + return true, ourInode, nil } break } - return false, ourTs, nil + return false, ourInode, nil } func syncDir(dir string) error { @@ -211,7 +204,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix)) if err != nil { if os.IsNotExist(err) { - err = errors.New("was not built: no .dep") + err = errors.New("was not built: no .rec") } goto Finish } @@ -233,7 +226,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } // Check if target is not modified externally - modified, tsPrev, err := isModified(cwd, redoDir, tgt) + modified, inodePrev, err := isModified(cwd, redoDir, tgt) if err != nil { lockRelease() return TgtErr{tgtOrig, err} @@ -247,12 +240,13 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { return nil } - // Start preparing .dep + // Start preparing .rec fdDep, err := tempfile(redoDir, tgt+DepSuffix) if err != nil { lockRelease() return TgtErr{tgtOrig, err} } + fdDepPath := fdDep.Name() cleanup := func() { lockRelease() fdDep.Close() @@ -275,82 +269,72 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { cleanup() return TgtErr{tgtOrig, errors.New("no .do found")} } - if err = writeDep(fdDep, cwd, doFile); err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } // Determine basename and DIRPREFIX + doFileRelPath := doFile ents := strings.Split(cwd, "/") ents = ents[len(ents)-upLevels:] dirPrefix := path.Join(ents...) cwdOrig := cwd for i := 0; i < upLevels; i++ { cwd = path.Join(cwd, "..") + doFileRelPath = path.Join("..", doFileRelPath) } cwd = path.Clean(cwd) + doFilePath := path.Join(cwd, doFile) basename := tgt runErr := RunErr{Tgt: tgtOrig} if strings.HasPrefix(doFile, "default.") { basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1] - runErr.DoFile = doFile + runErr.DoFile = doFileRelPath } + + if err = writeDep(fdDep, cwdOrig, doFileRelPath); err != nil { + cleanup() + return TgtErr{tgtOrig, err} + } + fdDep.Close() trace(CWait, "%s", runErr.Name()) - doFile = path.Base(doFile) // Prepare command line var cmdName string var args []string - if err = unix.Access(path.Join(cwd, doFile), unix.X_OK); err == nil { - // Ordinary executable file - cmdName = doFile + if err = unix.Access(doFilePath, unix.X_OK); err == nil { + cmdName = doFilePath args = make([]string, 0, 3) } else { - fd, err := os.Open(path.Join(cwd, doFile)) - if err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } - buf := make([]byte, 512) - n, err := fd.Read(buf) - if err != nil { - cleanup() - return TgtErr{tgtOrig, err} - } - if n > 3 && string(buf[:3]) == "#!/" { - // Shebanged - t := string(buf[2:n]) - nlIdx := strings.Index(t, "\n") - if nlIdx == -1 { - cleanup() - return TgtErr{tgtOrig, errors.New("not fully read shebang")} - } - args = strings.Split(t[:nlIdx], " ") - cmdName, args = args[0], args[1:] + cmdName = "/bin/sh" + if traced { + args = append(args, "-ex") } else { - // Shell - cmdName = "/bin/sh" - if traced { - args = append(args, "-ex") - } else { - args = append(args, "-e") - } + args = append(args, "-e") } args = append(args, doFile) } // Temporary file for stdout - fdStdout, err := tempfile(cwd, tgt) + fdStdout, err := tempfile(cwdOrig, tgt) if err != nil { cleanup() return TgtErr{tgtOrig, err} } - tmpPath := fdStdout.Name() + ".3" // and for $3 - args = append(args, tgt, basename, path.Base(tmpPath)) + stdoutPath := fdStdout.Name() + fdStdout.Close() + tmpPath := stdoutPath + ".3" // and for $3 + tmpPathRel, err := filepath.Rel(cwd, tmpPath) + if err != nil { + panic(err) + } + args = append( + args, + path.Join(dirPrefix, tgt), + path.Join(dirPrefix, basename), + tmpPathRel, + ) cmd := exec.Command(cmdName, args...) cmd.Dir = cwd - cmd.Stdout = fdStdout + // cmd.Stdin reads from /dev/null by default cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", EnvLevel, Level+1)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvDirPrefix, dirPrefix)) cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", EnvBuildUUID, BuildUUID)) @@ -361,8 +345,11 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { )) fdNum := 0 - cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum)) + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgts) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsFd, 3+fdNum)) + fdNum++ + cmd.ExtraFiles = append(cmd.ExtraFiles, FdOODTgtsLock) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvOODTgtsLockFd, 3+fdNum)) fdNum++ if FdStatus == nil { @@ -386,10 +373,6 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } // Preparing stderr - stderr, err := cmd.StderrPipe() - if err != nil { - panic(err) - } var fdStderr *os.File if StderrKeep { fdStderr, err = os.OpenFile( @@ -403,7 +386,10 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } fdStderr.Truncate(0) } - shCtx := fmt.Sprintf("sh: %s: %s %s [%s]", tgtOrig, cmdName, args, cwd) + shCtx := fmt.Sprintf( + "sh: %s: %s %s cwd:%s dirprefix:%s", + tgtOrig, cmdName, args, cwd, dirPrefix, + ) trace(CDebug, "%s", shCtx) Jobs.Add(1) @@ -412,6 +398,26 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { if FdStatus != nil { FdStatus.Write([]byte{StatusRun}) } + + started := time.Now() + runErr.Started = &started + fdStdout, err = os.OpenFile(stdoutPath, os.O_RDWR, os.FileMode(0666)) + if err != nil { + runErr.Err = err + errs <- runErr + return + } + cmd.Stdout = fdStdout + fdDep, err = os.OpenFile(fdDepPath, os.O_WRONLY|os.O_APPEND, os.FileMode(0666)) + if err != nil { + runErr.Err = err + errs <- runErr + return + } + cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", EnvDepFd, 3+fdNum)) + fdNum++ + defer func() { jsRelease(shCtx) lockRelease() @@ -429,9 +435,14 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } Jobs.Done() }() - started := time.Now() - runErr.Started = &started - err := cmd.Start() + stderr, err := cmd.StderrPipe() + if err != nil { + runErr.Err = err + errs <- runErr + return + } + started = time.Now() + err = cmd.Start() if err != nil { runErr.Err = err errs <- runErr @@ -444,7 +455,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { go func() { scanner := bufio.NewScanner(stderr) var line string - ts := new(TAI64N) + ts := new(tai64n.TAI64N) for scanner.Scan() { line = scanner.Text() if strings.HasPrefix(line, childStderrPrefix) { @@ -453,9 +464,9 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { continue } if fdStderr != nil { - tai64nNow(ts) + tai64n.FromTime(time.Now(), ts) LogMutex.Lock() - fmt.Fprintf(fdStderr, "@%s %s\n", hex.EncodeToString(ts[:]), line) + fmt.Fprintf(fdStderr, "%s %s\n", ts.Encode(), line) LogMutex.Unlock() } if StderrSilent { @@ -482,14 +493,15 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } // Was $1 touched? - if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { - ts, err := fileCtime(fd) - fd.Close() - if err == nil && ts != tsPrev { - runErr.Err = errors.New("$1 was explicitly touched") - errs <- runErr + if inodePrev != nil { + if fd, err := os.Open(path.Join(cwdOrig, tgt)); err == nil { + inode, err := inodeFromFile(fd) fd.Close() - return + if err == nil && !inode.Equals(inodePrev) { + runErr.Err = errors.New("$1 was explicitly touched") + errs <- runErr + return + } } } @@ -529,6 +541,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { // Do we need to ifcreate it, of ifchange with renaming? if fd == nil { + os.Remove(path.Join(cwdOrig, tgt)) err = ifcreate(fdDep, tgt) if err != nil { goto Finish @@ -556,7 +569,7 @@ func runScript(tgtOrig string, errs chan error, traced bool) error { } } - // Commit .dep + // Commit .rec if !NoSync { err = fdDep.Sync() if err != nil {