gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin = `<!DOCTYPE html>
  49 <html>
  50   <head>
  51     <title>Links for %s</title>
  52   </head>
  53   <body>
  54 `
  55         HTMLEnd      = "  </body>\n</html>\n"
  56         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
  57         SHA256Prefix = "sha256="
  58         SHA256Ext    = ".sha256"
  59         InternalFlag = ".internal"
  60         GPGSigExt    = ".asc"
  61         GPGSigAttr   = " data-gpg-sig=true"
  62
  63         Warranty = `This program is free software: you can redistribute it and/or modify
  64 it under the terms of the GNU General Public License as published by
  65 the Free Software Foundation, version 3 of the License.
  66
  67 This program is distributed in the hope that it will be useful,
  68 but WITHOUT ANY WARRANTY; without even the implied warranty of
  69 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  70 GNU General Public License for more details.
  71
  72 You should have received a copy of the GNU General Public License
  73 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  74 )
  75
  76 var (
  77         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  78         Version string = "UNKNOWN"
  79
  80         root             = flag.String("root", "./packages", "Path to packages directory")
  81         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  82         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  83         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  84         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  85         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  86         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
  87         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  88         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  89         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  90         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  91         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  92         version          = flag.Bool("version", false, "Print version information")
  93         warranty         = flag.Bool("warranty", false, "Print warranty information")
  94
  95         killed bool
  96
  97         normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+")
  98 )
  99
 100 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
 101         path := filepath.Join(*root, dir)
 102         if _, err := os.Stat(path); os.IsNotExist(err) {
 103                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 104                         http.Error(w, err.Error(), http.StatusInternalServerError)
 105                         return false
 106                 }
 107                 log.Println(r.RemoteAddr, "mkdir", dir)
 108         }
 109         return true
 110 }
 111
 112 func refreshDir(
 113         w http.ResponseWriter,
 114         r *http.Request,
 115         dir,
 116         filenameGet string,
 117         gpgUpdate bool,
 118 ) bool {
 119         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 120                 return true
 121         }
 122         resp, err := http.Get(*pypiURL + dir + "/")
 123         if err != nil {
 124                 http.Error(w, err.Error(), http.StatusBadGateway)
 125                 return false
 126         }
 127         body, err := ioutil.ReadAll(resp.Body)
 128         resp.Body.Close()
 129         if err != nil {
 130                 http.Error(w, err.Error(), http.StatusBadGateway)
 131                 return false
 132         }
 133         if !mkdirForPkg(w, r, dir) {
 134                 return false
 135         }
 136         dirPath := filepath.Join(*root, dir)
 137         var submatches []string
 138         var uri string
 139         var filename string
 140         var path string
 141         var pkgURL *url.URL
 142         var digest []byte
 143         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 144                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 145                 if len(submatches) == 0 {
 146                         continue
 147                 }
 148                 uri = submatches[1]
 149                 filename = submatches[2]
 150                 if pkgURL, err = url.Parse(uri); err != nil {
 151                         http.Error(w, err.Error(), http.StatusInternalServerError)
 152                         return false
 153                 }
 154                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 155                 if err != nil {
 156                         http.Error(w, err.Error(), http.StatusBadGateway)
 157                         return false
 158                 }
 159                 pkgURL.Fragment = ""
 160                 path = filepath.Join(dirPath, filename)
 161                 if filename == filenameGet {
 162                         if killed {
 163                                 // Skip heavy remote call, when shutting down
 164                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 165                                 return false
 166                         }
 167                         log.Println(r.RemoteAddr, "pypi download", filename)
 168                         resp, err = http.Get(pkgURL.String())
 169                         if err != nil {
 170                                 http.Error(w, err.Error(), http.StatusBadGateway)
 171                                 return false
 172                         }
 173                         defer resp.Body.Close()
 174                         hasher := sha256.New()
 175                         dst, err := TempFile(dirPath)
 176                         if err != nil {
 177                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 178                                 return false
 179                         }
 180                         wr := io.MultiWriter(hasher, dst)
 181                         if _, err = io.Copy(wr, resp.Body); err != nil {
 182                                 os.Remove(dst.Name())
 183                                 dst.Close()
 184                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 185                                 return false
 186                         }
 187                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 188                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 189                                 os.Remove(dst.Name())
 190                                 dst.Close()
 191                                 http.Error(w, err.Error(), http.StatusBadGateway)
 192                                 return false
 193                         }
 194                         if err = dst.Sync(); err != nil {
 195                                 os.Remove(dst.Name())
 196                                 dst.Close()
 197                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 198                                 return false
 199                         }
 200                         dst.Close()
 201                         if err = os.Rename(dst.Name(), path); err != nil {
 202                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 203                                 return false
 204                         }
 205                         if err = DirSync(dirPath); err != nil {
 206                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 207                                 return false
 208                         }
 209                 }
 210                 if filename == filenameGet || gpgUpdate {
 211                         if _, err = os.Stat(path); err == nil {
 212                                 if resp, err := http.Get(pkgURL.String() + GPGSigExt); err == nil {
 213                                         sig, err := ioutil.ReadAll(resp.Body)
 214                                         resp.Body.Close()
 215                                         if err == nil {
 216                                                 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 217                                                         http.Error(w, err.Error(), http.StatusInternalServerError)
 218                                                         return false
 219                                                 }
 220                                                 log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
 221                                         }
 222                                 }
 223                         }
 224                 }
 225                 path = path + SHA256Ext
 226                 _, err = os.Stat(path)
 227                 if err == nil {
 228                         continue
 229                 }
 230                 if !os.IsNotExist(err) {
 231                         http.Error(w, err.Error(), http.StatusInternalServerError)
 232                         return false
 233                 }
 234                 log.Println(r.RemoteAddr, "pypi touch", filename)
 235                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 236                         http.Error(w, err.Error(), http.StatusInternalServerError)
 237                         return false
 238                 }
 239         }
 240         return true
 241 }
 242
 243 func listRoot(w http.ResponseWriter, r *http.Request) {
 244         files, err := ioutil.ReadDir(*root)
 245         if err != nil {
 246                 http.Error(w, err.Error(), http.StatusInternalServerError)
 247                 return
 248         }
 249         var result bytes.Buffer
 250         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 251         for _, file := range files {
 252                 if file.Mode().IsDir() {
 253                         result.WriteString(fmt.Sprintf(
 254                                 HTMLElement,
 255                                 *refreshURLPath+file.Name()+"/",
 256                                 file.Name(),
 257                         ))
 258                 }
 259         }
 260         result.WriteString(HTMLEnd)
 261         w.Write(result.Bytes())
 262 }
 263
 264 func listDir(
 265         w http.ResponseWriter,
 266         r *http.Request,
 267         dir string,
 268         autorefresh,
 269         gpgUpdate bool,
 270 ) {
 271         dirPath := filepath.Join(*root, dir)
 272         if autorefresh {
 273                 if !refreshDir(w, r, dir, "", gpgUpdate) {
 274                         return
 275                 }
 276         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) {
 277                 return
 278         }
 279         files, err := ioutil.ReadDir(dirPath)
 280         if err != nil {
 281                 http.Error(w, err.Error(), http.StatusInternalServerError)
 282                 return
 283         }
 284         var result bytes.Buffer
 285         result.WriteString(fmt.Sprintf(HTMLBegin, dir))
 286         var data []byte
 287         var gpgSigAttr string
 288         var filenameClean string
 289         for _, file := range files {
 290                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 291                         continue
 292                 }
 293                 if killed {
 294                         // Skip expensive I/O when shutting down
 295                         http.Error(w, "shutting down", http.StatusInternalServerError)
 296                         return
 297                 }
 298                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 299                 if err != nil {
 300                         http.Error(w, err.Error(), http.StatusInternalServerError)
 301                         return
 302                 }
 303                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 304                 if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) {
 305                         gpgSigAttr = ""
 306                 } else {
 307                         gpgSigAttr = GPGSigAttr
 308                 }
 309                 result.WriteString(fmt.Sprintf(
 310                         HTMLElement,
 311                         strings.Join([]string{
 312                                 *refreshURLPath, dir, "/",
 313                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
 314                         }, ""),
 315                         gpgSigAttr,
 316                         filenameClean,
 317                 ))
 318         }
 319         result.WriteString(HTMLEnd)
 320         w.Write(result.Bytes())
 321 }
 322
 323 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 324         log.Println(r.RemoteAddr, "get", filename)
 325         path := filepath.Join(*root, dir, filename)
 326         if _, err := os.Stat(path); os.IsNotExist(err) {
 327                 if !refreshDir(w, r, dir, filename, false) {
 328                         return
 329                 }
 330         }
 331         http.ServeFile(w, r, path)
 332 }
 333
 334 func serveUpload(w http.ResponseWriter, r *http.Request) {
 335         // Authentication
 336         username, password, ok := r.BasicAuth()
 337         if !ok {
 338                 log.Println(r.RemoteAddr, "unauthenticated", username)
 339                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 340                 return
 341         }
 342         auther, ok := passwords[username]
 343         if !ok || !auther.Auth(password) {
 344                 log.Println(r.RemoteAddr, "unauthenticated", username)
 345                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 346                 return
 347         }
 348
 349         // Form parsing
 350         var err error
 351         if err = r.ParseMultipartForm(1 << 20); err != nil {
 352                 http.Error(w, err.Error(), http.StatusBadRequest)
 353                 return
 354         }
 355         pkgNames, exists := r.MultipartForm.Value["name"]
 356         if !exists || len(pkgNames) != 1 {
 357                 http.Error(w, "single name is expected in request", http.StatusBadRequest)
 358                 return
 359         }
 360         dir := normalizationRe.ReplaceAllString(pkgNames[0], "-")
 361         dirPath := filepath.Join(*root, dir)
 362         var digestExpected []byte
 363         if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists {
 364                 digestExpected, err = hex.DecodeString(digestExpectedHex[0])
 365                 if err != nil {
 366                         http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest)
 367                         return
 368                 }
 369         }
 370         gpgSigsExpected := make(map[string]struct{})
 371
 372         // Checking is it internal package
 373         if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil {
 374                 log.Println(r.RemoteAddr, "non-internal package", dir)
 375                 http.Error(w, "unknown internal package", http.StatusUnauthorized)
 376                 return
 377         }
 378
 379         for _, file := range r.MultipartForm.File["content"] {
 380                 filename := file.Filename
 381                 gpgSigsExpected[filename+GPGSigExt] = struct{}{}
 382                 log.Println(r.RemoteAddr, "put", filename, "by", username)
 383                 path := filepath.Join(dirPath, filename)
 384                 if _, err = os.Stat(path); err == nil {
 385                         log.Println(r.RemoteAddr, "already exists", filename)
 386                         http.Error(w, "already exists", http.StatusBadRequest)
 387                         return
 388                 }
 389                 if !mkdirForPkg(w, r, dir) {
 390                         return
 391                 }
 392                 src, err := file.Open()
 393                 defer src.Close()
 394                 if err != nil {
 395                         http.Error(w, err.Error(), http.StatusInternalServerError)
 396                         return
 397                 }
 398                 dst, err := TempFile(dirPath)
 399                 if err != nil {
 400                         http.Error(w, err.Error(), http.StatusInternalServerError)
 401                         return
 402                 }
 403                 hasher := sha256.New()
 404                 wr := io.MultiWriter(hasher, dst)
 405                 if _, err = io.Copy(wr, src); err != nil {
 406                         os.Remove(dst.Name())
 407                         dst.Close()
 408                         http.Error(w, err.Error(), http.StatusInternalServerError)
 409                         return
 410                 }
 411                 if err = dst.Sync(); err != nil {
 412                         os.Remove(dst.Name())
 413                         dst.Close()
 414                         http.Error(w, err.Error(), http.StatusInternalServerError)
 415                         return
 416                 }
 417                 dst.Close()
 418                 digest := hasher.Sum(nil)
 419                 if digestExpected != nil {
 420                         if bytes.Compare(digestExpected, digest) == 0 {
 421                                 log.Println(r.RemoteAddr, filename, "good checksum received")
 422                         } else {
 423                                 log.Println(r.RemoteAddr, filename, "bad checksum received")
 424                                 http.Error(w, "bad checksum", http.StatusBadRequest)
 425                                 os.Remove(dst.Name())
 426                                 return
 427                         }
 428                 }
 429                 if err = os.Rename(dst.Name(), path); err != nil {
 430                         http.Error(w, err.Error(), http.StatusInternalServerError)
 431                         return
 432                 }
 433                 if err = DirSync(dirPath); err != nil {
 434                         http.Error(w, err.Error(), http.StatusInternalServerError)
 435                         return
 436                 }
 437                 if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil {
 438                         http.Error(w, err.Error(), http.StatusInternalServerError)
 439                         return
 440                 }
 441         }
 442         for _, file := range r.MultipartForm.File["gpg_signature"] {
 443                 filename := file.Filename
 444                 if _, exists := gpgSigsExpected[filename]; !exists {
 445                         http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest)
 446                         return
 447                 }
 448                 delete(gpgSigsExpected, filename)
 449                 log.Println(r.RemoteAddr, "put", filename, "by", username)
 450                 path := filepath.Join(dirPath, filename)
 451                 if _, err = os.Stat(path); err == nil {
 452                         log.Println(r.RemoteAddr, "already exists", filename)
 453                         http.Error(w, "already exists", http.StatusBadRequest)
 454                         return
 455                 }
 456                 src, err := file.Open()
 457                 if err != nil {
 458                         http.Error(w, err.Error(), http.StatusInternalServerError)
 459                         return
 460                 }
 461                 sig, err := ioutil.ReadAll(src)
 462                 src.Close()
 463                 if err != nil {
 464                         http.Error(w, err.Error(), http.StatusInternalServerError)
 465                         return
 466                 }
 467                 if err = WriteFileSync(dirPath, path, sig); err != nil {
 468                         http.Error(w, err.Error(), http.StatusInternalServerError)
 469                         return
 470                 }
 471         }
 472 }
 473
 474 func handler(w http.ResponseWriter, r *http.Request) {
 475         switch r.Method {
 476         case "GET":
 477                 var path string
 478                 var autorefresh bool
 479                 var gpgUpdate bool
 480                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 481                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 482                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 483                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 484                         autorefresh = true
 485                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 486                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 487                         autorefresh = true
 488                         gpgUpdate = true
 489                 } else {
 490                         http.Error(w, "unknown action", http.StatusBadRequest)
 491                         return
 492                 }
 493                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 494                 if len(parts) > 2 {
 495                         http.Error(w, "invalid path", http.StatusBadRequest)
 496                         return
 497                 }
 498                 if len(parts) == 1 {
 499                         if parts[0] == "" {
 500                                 listRoot(w, r)
 501                         } else {
 502                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 503                         }
 504                 } else {
 505                         servePkg(w, r, parts[0], parts[1])
 506                 }
 507         case "POST":
 508                 serveUpload(w, r)
 509         default:
 510                 http.Error(w, "unknown action", http.StatusBadRequest)
 511         }
 512 }
 513
 514 func goodIntegrity() bool {
 515         dirs, err := ioutil.ReadDir(*root)
 516         if err != nil {
 517                 log.Fatal(err)
 518         }
 519         hasher := sha256.New()
 520         digest := make([]byte, sha256.Size)
 521         isGood := true
 522         var data []byte
 523         var pkgName string
 524         for _, dir := range dirs {
 525                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 526                 if err != nil {
 527                         log.Fatal(err)
 528                 }
 529                 for _, file := range files {
 530                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 531                                 continue
 532                         }
 533                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 534                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 535                         if err != nil {
 536                                 if os.IsNotExist(err) {
 537                                         continue
 538                                 }
 539                                 log.Fatal(err)
 540                         }
 541                         hasher.Write(data)
 542                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 543                         if err != nil {
 544                                 log.Fatal(err)
 545                         }
 546                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 547                                 log.Println(pkgName, "GOOD")
 548                         } else {
 549                                 isGood = false
 550                                 log.Println(pkgName, "BAD")
 551                         }
 552                         hasher.Reset()
 553                 }
 554         }
 555         return isGood
 556 }
 557
 558 func main() {
 559         flag.Parse()
 560         if *warranty {
 561                 fmt.Println(Warranty)
 562                 return
 563         }
 564         if *version {
 565                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 566                 return
 567         }
 568         if *fsck {
 569                 if !goodIntegrity() {
 570                         os.Exit(1)
 571                 }
 572                 return
 573         }
 574         if *passwdCheck {
 575                 refreshPasswd()
 576                 return
 577         }
 578         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 579                 log.Fatalln("Both -tls-cert and -tls-key are required")
 580         }
 581         refreshPasswd()
 582         log.Println("root:", *root, "bind:", *bind)
 583
 584         ln, err := net.Listen("tcp", *bind)
 585         if err != nil {
 586                 log.Fatal(err)
 587         }
 588         ln = netutil.LimitListener(ln, *maxClients)
 589         server := &http.Server{
 590                 ReadTimeout:  time.Minute,
 591                 WriteTimeout: time.Minute,
 592         }
 593         http.HandleFunc(*norefreshURLPath, handler)
 594         http.HandleFunc(*refreshURLPath, handler)
 595         http.HandleFunc(*gpgUpdateURLPath, handler)
 596
 597         needsRefreshPasswd := make(chan os.Signal, 0)
 598         needsShutdown := make(chan os.Signal, 0)
 599         exitErr := make(chan error, 0)
 600         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 601         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 602         go func() {
 603                 for range needsRefreshPasswd {
 604                         log.Println("Refreshing passwords")
 605                         refreshPasswd()
 606                 }
 607         }()
 608         go func(s *http.Server) {
 609                 <-needsShutdown
 610                 killed = true
 611                 log.Println("Shutting down")
 612                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 613                 exitErr <- s.Shutdown(ctx)
 614                 cancel()
 615         }(server)
 616
 617         if *tlsCert == "" {
 618                 err = server.Serve(ln)
 619         } else {
 620                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 621         }
 622         if err != http.ErrServerClosed {
 623                 log.Fatal(err)
 624         }
 625         if err := <-exitErr; err != nil {
 626                 log.Fatal(err)
 627         }
 628 }