gocheese.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
   4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
   5
   6 This program is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, version 3 of the License.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 // Python private package repository and caching proxy
  20 package main
  21
  22 import (
  23         "bytes"
  24         "context"
  25         "crypto/sha256"
  26         "encoding/hex"
  27         "flag"
  28         "fmt"
  29         "io"
  30         "io/ioutil"
  31         "log"
  32         "net"
  33         "net/http"
  34         "net/url"
  35         "os"
  36         "os/signal"
  37         "path/filepath"
  38         "regexp"
  39         "runtime"
  40         "strings"
  41         "syscall"
  42         "time"
  43
  44         "golang.org/x/net/netutil"
  45 )
  46
  47 const (
  48         HTMLBegin = `<!DOCTYPE html>
  49 <html>
  50   <head>
  51     <title>Links for %s</title>
  52   </head>
  53   <body>
  54 `
  55         HTMLEnd      = "  </body>\n</html>\n"
  56         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
  57         SHA256Prefix = "sha256="
  58         SHA256Ext    = ".sha256"
  59         InternalFlag = ".internal"
  60         GPGSigExt    = ".asc"
  61         GPGSigAttr   = " data-gpg-sig=true"
  62
  63         Warranty = `This program is free software: you can redistribute it and/or modify
  64 it under the terms of the GNU General Public License as published by
  65 the Free Software Foundation, version 3 of the License.
  66
  67 This program is distributed in the hope that it will be useful,
  68 but WITHOUT ANY WARRANTY; without even the implied warranty of
  69 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  70 GNU General Public License for more details.
  71
  72 You should have received a copy of the GNU General Public License
  73 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
  74 )
  75
  76 var (
  77         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
  78         Version string = "UNKNOWN"
  79
  80         root             = flag.String("root", "./packages", "Path to packages directory")
  81         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
  82         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
  83         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
  84         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
  85         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
  86         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
  87         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
  88         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
  89         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
  90         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
  91         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
  92         version          = flag.Bool("version", false, "Print version information")
  93         warranty         = flag.Bool("warranty", false, "Print warranty information")
  94
  95         killed bool
  96 )
  97
  98 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
  99         path := filepath.Join(*root, dir)
 100         if _, err := os.Stat(path); os.IsNotExist(err) {
 101                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
 102                         http.Error(w, err.Error(), http.StatusInternalServerError)
 103                         return false
 104                 }
 105                 log.Println(r.RemoteAddr, "mkdir", dir)
 106         }
 107         return true
 108 }
 109
 110 func refreshDir(
 111         w http.ResponseWriter,
 112         r *http.Request,
 113         dir,
 114         filenameGet string,
 115         gpgUpdate bool,
 116 ) bool {
 117         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
 118                 return true
 119         }
 120         resp, err := http.Get(*pypiURL + dir + "/")
 121         if err != nil {
 122                 http.Error(w, err.Error(), http.StatusBadGateway)
 123                 return false
 124         }
 125         body, err := ioutil.ReadAll(resp.Body)
 126         resp.Body.Close()
 127         if err != nil {
 128                 http.Error(w, err.Error(), http.StatusBadGateway)
 129                 return false
 130         }
 131         if !mkdirForPkg(w, r, dir) {
 132                 return false
 133         }
 134         dirPath := filepath.Join(*root, dir)
 135         var submatches []string
 136         var uri string
 137         var filename string
 138         var path string
 139         var pkgURL *url.URL
 140         var digest []byte
 141         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
 142                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
 143                 if len(submatches) == 0 {
 144                         continue
 145                 }
 146                 uri = submatches[1]
 147                 filename = submatches[2]
 148                 if pkgURL, err = url.Parse(uri); err != nil {
 149                         http.Error(w, err.Error(), http.StatusInternalServerError)
 150                         return false
 151                 }
 152                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
 153                 if err != nil {
 154                         http.Error(w, err.Error(), http.StatusBadGateway)
 155                         return false
 156                 }
 157                 pkgURL.Fragment = ""
 158                 path = filepath.Join(dirPath, filename)
 159                 if filename == filenameGet {
 160                         if killed {
 161                                 // Skip heavy remote call, when shutting down
 162                                 http.Error(w, "shutting down", http.StatusInternalServerError)
 163                                 return false
 164                         }
 165                         log.Println(r.RemoteAddr, "pypi download", filename)
 166                         resp, err = http.Get(pkgURL.String())
 167                         if err != nil {
 168                                 http.Error(w, err.Error(), http.StatusBadGateway)
 169                                 return false
 170                         }
 171                         defer resp.Body.Close()
 172                         hasher := sha256.New()
 173                         dst, err := TempFile(dirPath)
 174                         if err != nil {
 175                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 176                                 return false
 177                         }
 178                         wr := io.MultiWriter(hasher, dst)
 179                         if _, err = io.Copy(wr, resp.Body); err != nil {
 180                                 os.Remove(dst.Name())
 181                                 dst.Close()
 182                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 183                                 return false
 184                         }
 185                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
 186                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
 187                                 os.Remove(dst.Name())
 188                                 dst.Close()
 189                                 http.Error(w, err.Error(), http.StatusBadGateway)
 190                                 return false
 191                         }
 192                         if err = dst.Sync(); err != nil {
 193                                 os.Remove(dst.Name())
 194                                 dst.Close()
 195                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 196                                 return false
 197                         }
 198                         dst.Close()
 199                         if err = os.Rename(dst.Name(), path); err != nil {
 200                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 201                                 return false
 202                         }
 203                 }
 204                 if filename == filenameGet || gpgUpdate {
 205                         if _, err = os.Stat(path); err == nil {
 206                                 if resp, err := http.Get(pkgURL.String() + GPGSigExt); err == nil {
 207                                         sig, err := ioutil.ReadAll(resp.Body)
 208                                         resp.Body.Close()
 209                                         if err == nil {
 210                                                 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
 211                                                         http.Error(w, err.Error(), http.StatusInternalServerError)
 212                                                         return false
 213                                                 }
 214                                                 log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
 215                                         }
 216                                 }
 217                         }
 218                 }
 219                 path = path + SHA256Ext
 220                 _, err = os.Stat(path)
 221                 if err == nil {
 222                         continue
 223                 }
 224                 if !os.IsNotExist(err) {
 225                         http.Error(w, err.Error(), http.StatusInternalServerError)
 226                         return false
 227                 }
 228                 log.Println(r.RemoteAddr, "pypi touch", filename)
 229                 if err = WriteFileSync(dirPath, path, digest); err != nil {
 230                         http.Error(w, err.Error(), http.StatusInternalServerError)
 231                         return false
 232                 }
 233         }
 234         return true
 235 }
 236
 237 func listRoot(w http.ResponseWriter, r *http.Request) {
 238         files, err := ioutil.ReadDir(*root)
 239         if err != nil {
 240                 http.Error(w, err.Error(), http.StatusInternalServerError)
 241                 return
 242         }
 243         var result bytes.Buffer
 244         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
 245         for _, file := range files {
 246                 if file.Mode().IsDir() {
 247                         result.WriteString(fmt.Sprintf(
 248                                 HTMLElement,
 249                                 *refreshURLPath+file.Name()+"/",
 250                                 file.Name(),
 251                         ))
 252                 }
 253         }
 254         result.WriteString(HTMLEnd)
 255         w.Write(result.Bytes())
 256 }
 257
 258 func listDir(
 259         w http.ResponseWriter,
 260         r *http.Request,
 261         dir string,
 262         autorefresh,
 263         gpgUpdate bool,
 264 ) {
 265         dirPath := filepath.Join(*root, dir)
 266         if autorefresh {
 267                 if !refreshDir(w, r, dir, "", gpgUpdate) {
 268                         return
 269                 }
 270         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) {
 271                 return
 272         }
 273         files, err := ioutil.ReadDir(dirPath)
 274         if err != nil {
 275                 http.Error(w, err.Error(), http.StatusInternalServerError)
 276                 return
 277         }
 278         var result bytes.Buffer
 279         result.WriteString(fmt.Sprintf(HTMLBegin, dir))
 280         var data []byte
 281         var gpgSigAttr string
 282         var filenameClean string
 283         for _, file := range files {
 284                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
 285                         continue
 286                 }
 287                 if killed {
 288                         // Skip expensive I/O when shutting down
 289                         http.Error(w, "shutting down", http.StatusInternalServerError)
 290                         return
 291                 }
 292                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
 293                 if err != nil {
 294                         http.Error(w, err.Error(), http.StatusInternalServerError)
 295                         return
 296                 }
 297                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
 298                 if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) {
 299                         gpgSigAttr = ""
 300                 } else {
 301                         gpgSigAttr = GPGSigAttr
 302                 }
 303                 result.WriteString(fmt.Sprintf(
 304                         HTMLElement,
 305                         strings.Join([]string{
 306                                 *refreshURLPath, dir, "/",
 307                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
 308                         }, ""),
 309                         gpgSigAttr,
 310                         filenameClean,
 311                 ))
 312         }
 313         result.WriteString(HTMLEnd)
 314         w.Write(result.Bytes())
 315 }
 316
 317 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
 318         log.Println(r.RemoteAddr, "get", filename)
 319         path := filepath.Join(*root, dir, filename)
 320         if _, err := os.Stat(path); os.IsNotExist(err) {
 321                 if !refreshDir(w, r, dir, filename, false) {
 322                         return
 323                 }
 324         }
 325         http.ServeFile(w, r, path)
 326 }
 327
 328 func serveUpload(w http.ResponseWriter, r *http.Request) {
 329         username, password, ok := r.BasicAuth()
 330         if !ok {
 331                 log.Println(r.RemoteAddr, "unauthenticated", username)
 332                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 333                 return
 334         }
 335         auther, ok := passwords[username]
 336         if !ok || !auther.Auth(password) {
 337                 log.Println(r.RemoteAddr, "unauthenticated", username)
 338                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
 339                 return
 340         }
 341         var err error
 342         if err = r.ParseMultipartForm(1 << 20); err != nil {
 343                 http.Error(w, err.Error(), http.StatusBadRequest)
 344                 return
 345         }
 346         var digestExpected []byte
 347         if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists {
 348                 digestExpected, err = hex.DecodeString(digestExpectedHex[0])
 349                 if err != nil {
 350                         http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest)
 351                         return
 352                 }
 353         }
 354         gpgSigsExpected := make(map[string]struct{})
 355         for _, file := range r.MultipartForm.File["content"] {
 356                 filename := file.Filename
 357                 gpgSigsExpected[filename+GPGSigExt] = struct{}{}
 358                 log.Println(r.RemoteAddr, "put", filename, "by", username)
 359                 dir := filename[:strings.LastIndex(filename, "-")]
 360                 dirPath := filepath.Join(*root, dir)
 361                 path := filepath.Join(dirPath, filename)
 362                 if _, err = os.Stat(path); err == nil {
 363                         log.Println(r.RemoteAddr, "already exists", filename)
 364                         http.Error(w, "Already exists", http.StatusBadRequest)
 365                         return
 366                 }
 367                 if !mkdirForPkg(w, r, dir) {
 368                         return
 369                 }
 370                 internalPath := filepath.Join(dirPath, InternalFlag)
 371                 var dst *os.File
 372                 if _, err = os.Stat(internalPath); os.IsNotExist(err) {
 373                         if dst, err = os.Create(internalPath); err != nil {
 374                                 http.Error(w, err.Error(), http.StatusInternalServerError)
 375                                 return
 376                         }
 377                         dst.Close()
 378                 }
 379                 src, err := file.Open()
 380                 defer src.Close()
 381                 if err != nil {
 382                         http.Error(w, err.Error(), http.StatusInternalServerError)
 383                         return
 384                 }
 385                 dst, err = TempFile(dirPath)
 386                 if err != nil {
 387                         http.Error(w, err.Error(), http.StatusInternalServerError)
 388                         return
 389                 }
 390                 hasher := sha256.New()
 391                 wr := io.MultiWriter(hasher, dst)
 392                 if _, err = io.Copy(wr, src); err != nil {
 393                         os.Remove(dst.Name())
 394                         dst.Close()
 395                         http.Error(w, err.Error(), http.StatusInternalServerError)
 396                         return
 397                 }
 398                 if err = dst.Sync(); err != nil {
 399                         os.Remove(dst.Name())
 400                         dst.Close()
 401                         http.Error(w, err.Error(), http.StatusInternalServerError)
 402                         return
 403                 }
 404                 dst.Close()
 405                 digest := hasher.Sum(nil)
 406                 if digestExpected != nil {
 407                         if bytes.Compare(digestExpected, digest) == 0 {
 408                                 log.Println(r.RemoteAddr, filename, "good checksum received")
 409                         } else {
 410                                 log.Println(r.RemoteAddr, filename, "bad checksum received")
 411                                 http.Error(w, "bad checksum", http.StatusBadRequest)
 412                                 os.Remove(dst.Name())
 413                                 return
 414                         }
 415                 }
 416                 if err = os.Rename(dst.Name(), path); err != nil {
 417                         http.Error(w, err.Error(), http.StatusInternalServerError)
 418                         return
 419                 }
 420                 if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil {
 421                         http.Error(w, err.Error(), http.StatusInternalServerError)
 422                         return
 423                 }
 424         }
 425         for _, file := range r.MultipartForm.File["gpg_signature"] {
 426                 filename := file.Filename
 427                 if _, exists := gpgSigsExpected[filename]; !exists {
 428                         http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest)
 429                         return
 430                 }
 431                 delete(gpgSigsExpected, filename)
 432                 log.Println(r.RemoteAddr, "put", filename, "by", username)
 433                 dir := filename[:strings.LastIndex(filename, "-")]
 434                 dirPath := filepath.Join(*root, dir)
 435                 path := filepath.Join(dirPath, filename)
 436                 if _, err = os.Stat(path); err == nil {
 437                         log.Println(r.RemoteAddr, "already exists", filename)
 438                         http.Error(w, "Already exists", http.StatusBadRequest)
 439                         return
 440                 }
 441                 src, err := file.Open()
 442                 if err != nil {
 443                         http.Error(w, err.Error(), http.StatusInternalServerError)
 444                         return
 445                 }
 446                 sig, err := ioutil.ReadAll(src)
 447                 src.Close()
 448                 if err != nil {
 449                         http.Error(w, err.Error(), http.StatusInternalServerError)
 450                         return
 451                 }
 452                 if err = WriteFileSync(dirPath, path, sig); err != nil {
 453                         http.Error(w, err.Error(), http.StatusInternalServerError)
 454                         return
 455                 }
 456         }
 457 }
 458
 459 func handler(w http.ResponseWriter, r *http.Request) {
 460         switch r.Method {
 461         case "GET":
 462                 var path string
 463                 var autorefresh bool
 464                 var gpgUpdate bool
 465                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
 466                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
 467                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
 468                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
 469                         autorefresh = true
 470                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
 471                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
 472                         autorefresh = true
 473                         gpgUpdate = true
 474                 } else {
 475                         http.Error(w, "unknown action", http.StatusBadRequest)
 476                         return
 477                 }
 478                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
 479                 if len(parts) > 2 {
 480                         http.Error(w, "invalid path", http.StatusBadRequest)
 481                         return
 482                 }
 483                 if len(parts) == 1 {
 484                         if parts[0] == "" {
 485                                 listRoot(w, r)
 486                         } else {
 487                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
 488                         }
 489                 } else {
 490                         servePkg(w, r, parts[0], parts[1])
 491                 }
 492         case "POST":
 493                 serveUpload(w, r)
 494         default:
 495                 http.Error(w, "unknown action", http.StatusBadRequest)
 496         }
 497 }
 498
 499 func goodIntegrity() bool {
 500         dirs, err := ioutil.ReadDir(*root)
 501         if err != nil {
 502                 log.Fatal(err)
 503         }
 504         hasher := sha256.New()
 505         digest := make([]byte, sha256.Size)
 506         isGood := true
 507         var data []byte
 508         var pkgName string
 509         for _, dir := range dirs {
 510                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
 511                 if err != nil {
 512                         log.Fatal(err)
 513                 }
 514                 for _, file := range files {
 515                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
 516                                 continue
 517                         }
 518                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
 519                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
 520                         if err != nil {
 521                                 if os.IsNotExist(err) {
 522                                         continue
 523                                 }
 524                                 log.Fatal(err)
 525                         }
 526                         hasher.Write(data)
 527                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
 528                         if err != nil {
 529                                 log.Fatal(err)
 530                         }
 531                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
 532                                 log.Println(pkgName, "GOOD")
 533                         } else {
 534                                 isGood = false
 535                                 log.Println(pkgName, "BAD")
 536                         }
 537                         hasher.Reset()
 538                 }
 539         }
 540         return isGood
 541 }
 542
 543 func main() {
 544         flag.Parse()
 545         if *warranty {
 546                 fmt.Println(Warranty)
 547                 return
 548         }
 549         if *version {
 550                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
 551                 return
 552         }
 553         if *fsck {
 554                 if !goodIntegrity() {
 555                         os.Exit(1)
 556                 }
 557                 return
 558         }
 559         if *passwdCheck {
 560                 refreshPasswd()
 561                 return
 562         }
 563         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
 564                 log.Fatalln("Both -tls-cert and -tls-key are required")
 565         }
 566         refreshPasswd()
 567         log.Println("root:", *root, "bind:", *bind)
 568
 569         ln, err := net.Listen("tcp", *bind)
 570         if err != nil {
 571                 log.Fatal(err)
 572         }
 573         ln = netutil.LimitListener(ln, *maxClients)
 574         server := &http.Server{
 575                 ReadTimeout:  time.Minute,
 576                 WriteTimeout: time.Minute,
 577         }
 578         http.HandleFunc(*norefreshURLPath, handler)
 579         http.HandleFunc(*refreshURLPath, handler)
 580         http.HandleFunc(*gpgUpdateURLPath, handler)
 581
 582         needsRefreshPasswd := make(chan os.Signal, 0)
 583         needsShutdown := make(chan os.Signal, 0)
 584         exitErr := make(chan error, 0)
 585         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
 586         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
 587         go func() {
 588                 for range needsRefreshPasswd {
 589                         log.Println("Refreshing passwords")
 590                         refreshPasswd()
 591                 }
 592         }()
 593         go func(s *http.Server) {
 594                 <-needsShutdown
 595                 killed = true
 596                 log.Println("Shutting down")
 597                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
 598                 exitErr <- s.Shutdown(ctx)
 599                 cancel()
 600         }(server)
 601
 602         if *tlsCert == "" {
 603                 err = server.Serve(ln)
 604         } else {
 605                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
 606         }
 607         if err != http.ErrServerClosed {
 608                 log.Fatal(err)
 609         }
 610         if err := <-exitErr; err != nil {
 611                 log.Fatal(err)
 612         }
 613 }