run.go

   1 /*
   2 goredo -- redo implementation on pure Go
   3 Copyright (C) 2020 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 // Targets runner
  19
  20 package main
  21
  22 import (
  23         "bufio"
  24         "encoding/hex"
  25         "errors"
  26         "fmt"
  27         "io"
  28         "os"
  29         "os/exec"
  30         "path"
  31         "strconv"
  32         "strings"
  33         "syscall"
  34         "time"
  35
  36         "go.cypherpunks.ru/recfile"
  37         "golang.org/x/sys/unix"
  38 )
  39
  40 const (
  41         RedoDir    = ".redo"
  42         LockSuffix = ".lock"
  43         DepSuffix  = ".dep"
  44         TmpPrefix  = ".redo."
  45         LogSuffix  = ".log"
  46 )
  47
  48 func mkdirs(pth string) error {
  49         if _, err := os.Stat(pth); err == nil {
  50                 return nil
  51         }
  52         return os.MkdirAll(pth, os.FileMode(0777))
  53 }
  54
  55 func tempsuffix() string {
  56         return strconv.FormatInt((time.Now().UnixNano()+int64(os.Getpid()))&0xFFFFFFFF, 16)
  57 }
  58
  59 func tempfile(dir, prefix string) (*os.File, error) {
  60         // It respects umask, unlike ioutil.TempFile
  61         name := path.Join(dir, TmpPrefix+prefix+"."+tempsuffix())
  62         return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
  63 }
  64
  65 func isModified(cwd, redoDir, tgt string) (bool, error) {
  66         fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
  67         if err != nil {
  68                 if os.IsNotExist(err) {
  69                         return false, nil
  70                 }
  71                 return false, err
  72         }
  73         defer fdDep.Close()
  74         r := recfile.NewReader(fdDep)
  75         for {
  76                 m, err := r.NextMap()
  77                 if err != nil {
  78                         if err == io.EOF {
  79                                 break
  80                         }
  81                         return false, err
  82                 }
  83                 if m["Target"] != tgt {
  84                         continue
  85                 }
  86                 fd, err := os.Open(path.Join(cwd, tgt))
  87                 if err != nil {
  88                         if os.IsNotExist(err) {
  89                                 return false, nil
  90                         }
  91                         return false, err
  92                 }
  93                 defer fd.Close()
  94                 ourTs, err := fileCtime(fd)
  95                 if err != nil {
  96                         return false, err
  97                 }
  98                 if ourTs != m["Ctime"] {
  99                         return true, nil
 100                 }
 101                 break
 102         }
 103         return false, nil
 104 }
 105
 106 func runScript(tgt string, errs chan error) error {
 107         tgtOrig := tgt
 108         cwd, tgt := cwdAndTgt(tgt)
 109         redoDir := path.Join(cwd, RedoDir)
 110         errf := func(err error) error {
 111                 return fmt.Errorf("%s: %s", tgtOrig, err)
 112         }
 113         if err := mkdirs(redoDir); err != nil {
 114                 return errf(err)
 115         }
 116
 117         // Acquire lock
 118         fdLock, err := os.OpenFile(
 119                 path.Join(redoDir, "lock."+tgt),
 120                 os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
 121                 os.FileMode(0666),
 122         )
 123         if err != nil {
 124                 return errf(err)
 125         }
 126         lockRelease := func() {
 127                 trace(CLock, "LOCK_UN: %s", fdLock.Name())
 128                 unix.Flock(int(fdLock.Fd()), unix.LOCK_UN)
 129                 fdLock.Close()
 130         }
 131         trace(CLock, "LOCK_NB: %s", fdLock.Name())
 132         if err = unix.Flock(int(fdLock.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil {
 133                 if uintptr(err.(syscall.Errno)) != uintptr(unix.EWOULDBLOCK) {
 134                         fdLock.Close()
 135                         return errf(err)
 136                 }
 137                 trace(CDebug, "waiting: %s", tgtOrig)
 138                 Jobs.Add(1)
 139                 go func() {
 140                         defer Jobs.Done()
 141                         trace(CLock, "LOCK_EX: %s", fdLock.Name())
 142                         unix.Flock(int(fdLock.Fd()), unix.LOCK_EX)
 143                         lockRelease()
 144                         trace(CDebug, "waiting done: %s", tgtOrig)
 145                         var builtNow bool
 146                         fdDep, err := os.Open(path.Join(redoDir, tgt+DepSuffix))
 147                         if err != nil {
 148                                 if os.IsNotExist(err) {
 149                                         err = fmt.Errorf("%s is not built", tgtOrig)
 150                                 }
 151                                 goto Finish
 152                         }
 153                         builtNow, _, err = isBuiltNow(fdDep)
 154                         if err != nil {
 155                                 goto Finish
 156                         }
 157                         if !builtNow {
 158                                 err = fmt.Errorf("%s is not built", tgtOrig)
 159                         }
 160                 Finish:
 161                         if err != nil {
 162                                 err = errf(err)
 163                         }
 164                         errs <- err
 165                 }()
 166                 return nil
 167         }
 168
 169         // Check if target is not modified externally
 170         modified, err := isModified(cwd, redoDir, tgt)
 171         if err != nil {
 172                 lockRelease()
 173                 return errf(err)
 174         }
 175         if modified {
 176                 trace(CWarn, "%s externally modified: not redoing", tgtOrig)
 177                 lockRelease()
 178                 Jobs.Add(1)
 179                 go func() {
 180                         errs <- nil
 181                         Jobs.Done()
 182                 }()
 183                 return nil
 184         }
 185
 186         // Start preparing dep.
 187         fdDep, err := tempfile(redoDir, tgt+DepSuffix)
 188         if err != nil {
 189                 lockRelease()
 190                 return errf(err)
 191         }
 192         cleanup := func() {
 193                 lockRelease()
 194                 fdDep.Close()
 195                 os.Remove(fdDep.Name())
 196         }
 197         if _, err = recfile.NewWriter(fdDep).WriteFields(
 198                 recfile.Field{Name: "Build", Value: BuildUUID},
 199         ); err != nil {
 200                 cleanup()
 201                 return errf(err)
 202         }
 203
 204         // Find .do
 205         doFile, upLevels, err := findDo(fdDep, cwd, tgt)
 206         if err != nil {
 207                 cleanup()
 208                 return errf(err)
 209         }
 210         if doFile == "" {
 211                 cleanup()
 212                 return errf(errors.New("no .do found"))
 213         }
 214         if err = writeDep(fdDep, cwd, doFile); err != nil {
 215                 cleanup()
 216                 return errf(err)
 217         }
 218
 219         // Determine basename and DIRPREFIX
 220         ents := strings.Split(cwd, "/")
 221         ents = ents[len(ents)-upLevels:]
 222         dirPrefix := path.Join(ents...)
 223         cwdOrig := cwd
 224         for i := 0; i < upLevels; i++ {
 225                 cwd = path.Join(cwd, "..")
 226         }
 227         cwd = path.Clean(cwd)
 228         basename := tgt
 229         if strings.HasPrefix(doFile, "default.") {
 230                 basename = tgt[:len(tgt)-(len(doFile)-len("default.")-len(".do"))-1]
 231                 trace(CRedo, "%s (%s)", tgtOrig, doFile)
 232         } else {
 233                 trace(CRedo, "%s", tgtOrig)
 234         }
 235         doFile = path.Base(doFile)
 236
 237         // Prepare command line
 238         var cmdName string
 239         var args []string
 240         if err = unix.Access(path.Join(cwd, doFile), unix.X_OK); err == nil {
 241                 // Ordinary executable file
 242                 cmdName = doFile
 243                 args = make([]string, 0, 3)
 244         } else {
 245                 fd, err := os.Open(path.Join(cwd, doFile))
 246                 if err != nil {
 247                         cleanup()
 248                         return errf(err)
 249                 }
 250                 buf := make([]byte, 512)
 251                 n, err := fd.Read(buf)
 252                 if err != nil {
 253                         cleanup()
 254                         return errf(err)
 255                 }
 256                 if n > 3 && string(buf[:3]) == "#!/" {
 257                         // Shebanged
 258                         t := string(buf[2:n])
 259                         nlIdx := strings.Index(t, "\n")
 260                         if nlIdx == -1 {
 261                                 cleanup()
 262                                 return errf(errors.New("not fully read shebang"))
 263                         }
 264                         args = strings.Split(t[:nlIdx], " ")
 265                         cmdName, args = args[0], args[1:]
 266                 } else {
 267                         // Shell
 268                         cmdName = "/bin/sh"
 269                         if Trace {
 270                                 args = append(args, "-ex")
 271                         } else {
 272                                 args = append(args, "-e")
 273                         }
 274                 }
 275                 args = append(args, doFile)
 276         }
 277
 278         // Temporary file for stdout
 279         fdStdout, err := tempfile(cwd, tgt)
 280         if err != nil {
 281                 cleanup()
 282                 return errf(err)
 283         }
 284         tmpPath := fdStdout.Name() + ".3" // and for $3
 285         args = append(args, tgt, basename, path.Base(tmpPath))
 286
 287         cmd := exec.Command(cmdName, args...)
 288         cmd.Dir = cwd
 289         cmd.Stdout = fdStdout
 290         cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%d", RedoLevelEnv, Level+1))
 291         cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoDirPrefixEnv, dirPrefix))
 292         cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoBuildUUIDEnv, BuildUUID))
 293         childStderrPrefix := tempsuffix()
 294         cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", RedoStderrPrefixEnv, childStderrPrefix))
 295         cmd.ExtraFiles = append(cmd.ExtraFiles, fdDep)
 296         cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoDepFdEnv, 3+0))
 297         if JSR == nil {
 298                 // infinite jobs
 299                 cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSRFdEnv))
 300                 cmd.Env = append(cmd.Env, fmt.Sprintf("%s=NO", RedoJSWFdEnv))
 301         } else {
 302                 cmd.ExtraFiles = append(cmd.ExtraFiles, JSR)
 303                 cmd.ExtraFiles = append(cmd.ExtraFiles, JSW)
 304                 cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSRFdEnv, 3+1))
 305                 cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%d", RedoJSWFdEnv, 3+2))
 306         }
 307
 308         // Preparing stderr
 309         stderr, err := cmd.StderrPipe()
 310         if err != nil {
 311                 panic(err)
 312         }
 313         var fdStderr *os.File
 314         if StderrKeep {
 315                 fdStderr, err = os.OpenFile(
 316                         path.Join(redoDir, tgt+LogSuffix),
 317                         os.O_WRONLY|os.O_CREATE,
 318                         os.FileMode(0666),
 319                 )
 320                 if err != nil {
 321                         cleanup()
 322                         return errf(err)
 323                 }
 324                 fdStderr.Truncate(0)
 325         }
 326         trace(CDebug, "sh: %s: %s %s [%s]", tgtOrig, cmdName, args, cwd)
 327
 328         Jobs.Add(1)
 329         go func() {
 330                 jsAcquire()
 331                 started := time.Now()
 332                 defer func() {
 333                         jsRelease()
 334                         lockRelease()
 335                         fdDep.Close()
 336                         fdStdout.Close()
 337                         if fdStderr != nil {
 338                                 fdStderr.Close()
 339                         }
 340                         os.Remove(fdDep.Name())
 341                         os.Remove(fdStdout.Name())
 342                         os.Remove(tmpPath)
 343                         os.Remove(fdLock.Name())
 344                         finished := time.Now()
 345                         trace(CDone, "%s (%fsec)", tgtOrig, finished.Sub(started).Seconds())
 346                         Jobs.Done()
 347                 }()
 348                 err := cmd.Start()
 349                 if err != nil {
 350                         errs <- errf(err)
 351                         return
 352                 }
 353                 pid := fmt.Sprintf("[%d]", cmd.Process.Pid)
 354                 trace(CDebug, "%s runs %s", tgtOrig, pid)
 355
 356                 stderrTerm := make(chan struct{}, 0)
 357                 go func() {
 358                         scanner := bufio.NewScanner(stderr)
 359                         var line string
 360                         ts := new(TAI64N)
 361                         for scanner.Scan() {
 362                                 line = scanner.Text()
 363                                 if strings.HasPrefix(line, childStderrPrefix) {
 364                                         line = line[len(childStderrPrefix):]
 365                                         os.Stderr.WriteString(StderrPrefix + line + "\n")
 366                                         continue
 367                                 }
 368                                 if fdStderr != nil {
 369                                         tai64nNow(ts)
 370                                         fmt.Fprintf(fdStderr, "@%s %s\n", hex.EncodeToString(ts[:]), line)
 371                                 }
 372                                 if StderrSilent {
 373                                         continue
 374                                 }
 375                                 if MyPid == 0 {
 376                                         trace(CNone, "%s", line)
 377                                 } else {
 378                                         trace(CNone, "%s %s", pid, line)
 379                                 }
 380                         }
 381                         close(stderrTerm)
 382                 }()
 383
 384                 // Wait for job completion
 385                 <-stderrTerm
 386                 err = cmd.Wait()
 387                 if err != nil {
 388                         errs <- errf(err)
 389                         return
 390                 }
 391
 392                 // Does it produce both stdout and tmp?
 393                 fiStdout, err := os.Stat(fdStdout.Name())
 394                 if err != nil {
 395                         errs <- errf(err)
 396                         return
 397                 }
 398                 tmpExists := false
 399                 _, err = os.Stat(tmpPath)
 400                 if err == nil {
 401                         if fiStdout.Size() > 0 {
 402                                 errs <- errf(fmt.Errorf("%s created both tmp and stdout", tgtOrig))
 403                                 return
 404                         }
 405                         tmpExists = true
 406                 } else if !os.IsNotExist(err) {
 407                         errs <- errf(err)
 408                         return
 409                 }
 410
 411                 // Determine what file we must process at last
 412                 var fd *os.File
 413                 if tmpExists {
 414                         fd, err = os.Open(tmpPath)
 415                         if err != nil {
 416                                 errs <- errf(err)
 417                                 return
 418                         }
 419                         defer fd.Close()
 420                 } else if fiStdout.Size() > 0 {
 421                         fd = fdStdout
 422                 }
 423
 424                 // Do we need to ifcreate it, of ifchange with renaming?
 425                 if fd == nil {
 426                         if err = ifcreate(fdDep, tgt); err != nil {
 427                                 errs <- errf(err)
 428                                 return
 429                         }
 430                 } else {
 431                         if !NoSync {
 432                                 if err = fd.Sync(); err != nil {
 433                                         errs <- errf(err)
 434                                         return
 435                                 }
 436                         }
 437                         if err = os.Rename(fd.Name(), path.Join(cwdOrig, tgt)); err != nil {
 438                                 errs <- errf(err)
 439                                 return
 440                         }
 441                         if !NoSync {
 442                                 fdDir, err := os.Open(cwdOrig)
 443                                 if err != nil {
 444                                         errs <- errf(err)
 445                                         return
 446                                 }
 447                                 defer fdDir.Close()
 448                                 if err = fdDir.Sync(); err != nil {
 449                                         errs <- errf(err)
 450                                         return
 451                                 }
 452                         }
 453                         if err = writeDep(fdDep, cwdOrig, tgt); err != nil {
 454                                 errs <- errf(err)
 455                                 return
 456                         }
 457                 }
 458
 459                 // Commit dep.
 460                 if !NoSync {
 461                         if err = fdDep.Sync(); err != nil {
 462                                 errs <- errf(err)
 463                                 return
 464                         }
 465                 }
 466                 if err = os.Rename(fdDep.Name(), path.Join(redoDir, tgt+DepSuffix)); err != nil {
 467                         errs <- errf(err)
 468                         return
 469                 }
 470                 if !NoSync {
 471                         fdDir, err := os.Open(redoDir)
 472                         if err != nil {
 473                                 errs <- errf(err)
 474                                 return
 475                         }
 476                         defer fdDir.Close()
 477                         if err = fdDir.Sync(); err != nil {
 478                                 errs <- errf(err)
 479                                 return
 480                         }
 481                 }
 482                 errs <- nil
 483         }()
 484         return nil
 485 }